Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Import IGitt Data #100

Merged
merged 1 commit into from
Aug 2, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
data/: Import IGitt data
This create models for issues and mrs and import
related data for the same from the webservices.

Closes #83
  • Loading branch information
shrikrishna committed Aug 1, 2018

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 537602ecbe67263a1c3a877f88444ec6620190f4
2 changes: 2 additions & 0 deletions .ci/build.sh
Original file line number Diff line number Diff line change
@@ -24,6 +24,8 @@ python manage.py fetch_deployed_data _site $ISSUES_JSON --repo-name gh-board
python manage.py migrate
python manage.py test
python manage.py import_contributors_data
python manage.py import_issues_data
python manage.py import_merge_requests_data
python manage.py import_openhub_data

if [[ -f "_site/$META_REVIEW_DATA" ]]; then
88 changes: 88 additions & 0 deletions data/issues.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import logging
from dateutil.parser import parse
import pytz

import requests

from data.models import (
Issue,
Label,
Contributor,
)
from data.newcomers import active_newcomers
from data.webservices import webservices_url


def fetch_issues(hoster):
"""
Get issues opened by newcomers.

:param hoster: a string representing hoster, e.g. 'GitHub'
:return: a json of issues data
"""
logger = logging.getLogger(__name__)
hoster = hoster.lower()
import_url = webservices_url('issues/%s/all' % hoster)

headers = {'Content-Type': 'application/json'}
try:
response = requests.get(
url=import_url,
headers=headers,
)
response.raise_for_status()
except Exception as e:
logger.error(e)
return
issues = response.json()

# Removing issues which are not opened by newcomers
issues_list = []
for issue in issues:
if issue['author'] in active_newcomers():
issues_list.append(issue)
return issues_list


def import_issue(hoster, issue):
"""
Import issue data to database.

:param hoster: a string representing hoster
:param issue: a dict containing issue's data
"""
logger = logging.getLogger(__name__)
number = issue.get('number')
assignees = issue.pop('assignees')
labels = issue.pop('labels')
author = issue.pop('author')

# Parse string datetime to datetime object and add timezone support
issue['created_at'] = pytz.utc.localize(parse(issue['created_at']))
issue['updated_at'] = pytz.utc.localize(parse(issue['updated_at']))

try:
author, created = Contributor.objects.get_or_create(login=author)
issue['author'] = author
issue['hoster'] = hoster
issue_object, created = Issue.objects.get_or_create(**issue)

# Saving assignees
assignee_objects_list = []
for assignee in assignees:
assignee_object, created = Contributor.objects.get_or_create(
login=assignee)
assignee_objects_list.append(assignee_object)
issue_object.assignees.add(*assignee_objects_list)

# Saving labels
label_objects_list = []
for label in labels:
label_object, created = Label.objects.get_or_create(name=label)
label_objects_list.append(label_object)
issue_object.labels.add(*label_objects_list)
logger.info('Issue: %s has been saved.' % issue_object)
except Exception as ex:
logger.error(
'Something went wrong saving this issue %s: %s'
% (number, ex))
17 changes: 17 additions & 0 deletions data/management/commands/import_issues_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
from django.core.management.base import BaseCommand

from data.issues import fetch_issues, import_issue


class Command(BaseCommand):
help = 'Import issues opened by newcomers'

def handle(self, *args, **options):
github_data = fetch_issues('GitHub')
gitlab_data = fetch_issues('GitLab')
if github_data:
for data in github_data:
import_issue('GitHub', data)
if gitlab_data:
for data in gitlab_data:
import_issue('GitLab', data)
17 changes: 17 additions & 0 deletions data/management/commands/import_merge_requests_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
from django.core.management.base import BaseCommand

from data.merge_requests import fetch_mrs, import_mr


class Command(BaseCommand):
help = 'Import mrs opened by newcomers'

def handle(self, *args, **options):
github_data = fetch_mrs('GitHub')
gitlab_data = fetch_mrs('GitLab')
if github_data:
for data in github_data:
import_mr('GitHub', data)
if gitlab_data:
for data in gitlab_data:
import_mr('GitLab', data)
101 changes: 101 additions & 0 deletions data/merge_requests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
import logging
from dateutil.parser import parse
import pytz

import requests

from data.models import (
MergeRequest,
Label,
IssueNumber,
Contributor,
)
from data.newcomers import active_newcomers
from data.webservices import webservices_url


def fetch_mrs(hoster):
"""
Get mrs opened by newcomers.

:param hoster: a string representing hoster, e.g. 'GitHub'
:return: a json of mrs data
"""
logger = logging.getLogger(__name__)
hoster = hoster.lower()
import_url = webservices_url('mrs/%s/all' % hoster)

headers = {'Content-Type': 'application/json'}
try:
response = requests.get(
url=import_url,
headers=headers,
)
response.raise_for_status()
except Exception as e:
logger.error(e)
return
mrs = response.json()

# Removing mrs which are not opened by newcomers
mrs_list = []
for mr in mrs:
if mr['author'] in active_newcomers():
mrs_list.append(mr)
return mrs_list


def import_mr(hoster, mr):
"""
Import mr data to database.

:param hoster: a string representing hoster
:param mr: a dict containing mr's data
"""
logger = logging.getLogger(__name__)
number = mr.get('number')
assignees = mr.pop('assignees')
labels = mr.pop('labels')
author = mr.pop('author')
repo_id = mr['repo_id']
closes_issues = mr.pop('closes_issues')

# Parse string datetime to datetime object and add timezone support
mr['created_at'] = pytz.utc.localize(parse(mr['created_at']))
mr['updated_at'] = pytz.utc.localize(parse(mr['updated_at']))

try:
author, created = Contributor.objects.get_or_create(login=author)
mr['author'] = author
mr['hoster'] = hoster
mr_object, created = MergeRequest.objects.get_or_create(**mr)

# Saving assignees
assignee_objects_list = []
for assignee in assignees:
assignee_object, created = Contributor.objects.get_or_create(
login=assignee)
assignee_objects_list.append(assignee_object)
mr_object.assignees.add(*assignee_objects_list)

# Saving issues closes by this mr
closes_issues_list = []
for i_number in closes_issues:
issue_number_object, created = (
IssueNumber.objects.get_or_create(
number=i_number,
repo_id=repo_id))
closes_issues_list.append(issue_number_object)
mr_object.closes_issues.add(*closes_issues_list)

# Saving labels on the mr
label_objects_list = []
for label in labels:
label_object, created = Label.objects.get_or_create(name=label)
label_objects_list.append(label_object)
mr_object.labels.add(*label_objects_list)
logger.info('MR: %s has been saved.' % mr_object)
except Exception as ex:
logger.error(
'Something went wrong saving this mr %s: %s'
% (number, ex))
71 changes: 71 additions & 0 deletions data/migrations/0003_auto_20180801_0456.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.11 on 2018-08-01 04:56
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

dependencies = [
('data', '0002_auto_20180704_1130'),
]

operations = [
migrations.CreateModel(
name='Issue',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('number', models.IntegerField()),
('title', models.TextField()),
('repo', models.CharField(default=None, max_length=100, null=True)),
('repo_id', models.IntegerField()),
('created_at', models.DateTimeField()),
('updated_at', models.DateTimeField()),
('state', models.CharField(max_length=100)),
('hoster', models.CharField(max_length=100)),
('assignees', models.ManyToManyField(blank=True, related_name='issue_assignees', to='data.Contributor')),
('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='issue_author', to='data.Contributor')),
],
),
migrations.CreateModel(
name='IssueNumber',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('number', models.IntegerField()),
('repo_id', models.IntegerField()),
],
),
migrations.CreateModel(
name='Label',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=300)),
],
),
migrations.CreateModel(
name='MergeRequest',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('number', models.IntegerField()),
('title', models.TextField()),
('repo_id', models.IntegerField()),
('repo', models.CharField(default=None, max_length=100, null=True)),
('state', models.CharField(max_length=100)),
('created_at', models.DateTimeField()),
('updated_at', models.DateTimeField()),
('ci_status', models.BooleanField()),
('hoster', models.CharField(max_length=100)),
('assignees', models.ManyToManyField(blank=True, related_name='mr_assignees', to='data.Contributor')),
('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mr_author', to='data.Contributor')),
('closes_issues', models.ManyToManyField(blank=True, to='data.IssueNumber')),
('labels', models.ManyToManyField(blank=True, to='data.Label')),
],
),
migrations.AddField(
model_name='issue',
name='labels',
field=models.ManyToManyField(blank=True, to='data.Label'),
),
]
79 changes: 79 additions & 0 deletions data/models.py
Original file line number Diff line number Diff line change
@@ -22,3 +22,82 @@ def __str__(self):

class Meta:
ordering = ['login']


class Label(models.Model):
name = models.CharField(max_length=300)

def __str__(self):
return self.name


class Issue(models.Model):
number = models.IntegerField()
title = models.TextField()
repo = models.CharField(max_length=100, default=None, null=True)
repo_id = models.IntegerField()
author = models.ForeignKey(Contributor,
on_delete=models.CASCADE,
related_name='issue_author')
created_at = models.DateTimeField()
updated_at = models.DateTimeField()
state = models.CharField(max_length=100)
labels = models.ManyToManyField(Label, blank=True)
assignees = models.ManyToManyField(Contributor,
related_name='issue_assignees',
blank=True)
hoster = models.CharField(max_length=100)

def __str__(self):
return str(self.title)


class IssueNumber(models.Model):
number = models.IntegerField()
repo_id = models.IntegerField()

def __str__(self):
return str(self.number)

def get_issue(self):
"""
Get an issue object which number mathes with the
issue number and has same repo_id.
"""
issue = Issue.objects.get(number=self.number,
repo_id=self.repo_id)
return issue


class MergeRequest(models.Model):
number = models.IntegerField()
title = models.TextField()
repo_id = models.IntegerField()
repo = models.CharField(max_length=100, default=None, null=True)
closes_issues = models.ManyToManyField(IssueNumber, blank=True)
state = models.CharField(max_length=100)
author = models.ForeignKey(Contributor,
on_delete=models.CASCADE,
related_name='mr_author')
created_at = models.DateTimeField()
updated_at = models.DateTimeField()
assignees = models.ManyToManyField(Contributor,
related_name='mr_assignees',
blank=True)
ci_status = models.BooleanField()
labels = models.ManyToManyField(Label, blank=True)
hoster = models.CharField(max_length=100)

def __str__(self):
return self.title

def get_closes_issues_object(self):
"""
Get the list of issues object this mr is closing.
"""
issues_object_list = []
issue_numbers = self.closes_issues.all()
for issue_number in issue_numbers:
issue_object = issue_number.get_issue()
issues_object_list.append(issue_object)
return issues_object_list
Loading