data/: Import IGitt data

This create models for issues and mrs and import
related data for the same from the webservices.

Closes #83

.ci/build.sh

@@ -24,6 +24,8 @@ python manage.py fetch_deployed_data _site $ISSUES_JSON --repo-name gh-board
 python manage.py migrate
 python manage.py test
 python manage.py import_contributors_data
+python manage.py import_issues_data
+python manage.py import_merge_requests_data
 python manage.py import_openhub_data
 
 if [[ -f "_site/$META_REVIEW_DATA" ]]; then

data/issues.py

@@ -0,0 +1,88 @@
+import logging
+from dateutil.parser import parse
+import pytz
+
+import requests
+
+from data.models import (
+    Issue,
+    Label,
+    Contributor,
+    )
+from data.newcomers import active_newcomers
+from data.webservices import webservices_url
+
+
+def fetch_issues(hoster):
+    """
+    Get issues opened by newcomers.
+
+    :param hoster: a string representing hoster, e.g. 'GitHub'
+    :return: a json of issues data
+    """
+    logger = logging.getLogger(__name__)
+    hoster = hoster.lower()
+    import_url = webservices_url('issues/%s/all' % hoster)
+
+    headers = {'Content-Type': 'application/json'}
+    try:
+        response = requests.get(
+            url=import_url,
+            headers=headers,
+        )
+        response.raise_for_status()
+    except Exception as e:
+        logger.error(e)
+        return
+    issues = response.json()
+
+    # Removing issues which are not opened by newcomers
+    issues_list = []
+    for issue in issues:
+        if issue['author'] in active_newcomers():
+            issues_list.append(issue)
+    return issues_list
+
+
+def import_issue(hoster, issue):
+    """
+    Import issue data to database.
+
+    :param hoster: a string representing hoster
+    :param issue: a dict containing issue's data
+    """
+    logger = logging.getLogger(__name__)
+    number = issue.get('number')
+    assignees = issue.pop('assignees')
+    labels = issue.pop('labels')
+    author = issue.pop('author')
+
+    # Parse string datetime to datetime object and add timezone support
+    issue['created_at'] = pytz.utc.localize(parse(issue['created_at']))
+    issue['updated_at'] = pytz.utc.localize(parse(issue['updated_at']))
+
+    try:
+        author, created = Contributor.objects.get_or_create(login=author)
+        issue['author'] = author
+        issue['hoster'] = hoster
+        issue_object, created = Issue.objects.get_or_create(**issue)
+
+        # Saving assignees
+        assignee_objects_list = []
+        for assignee in assignees:
+            assignee_object, created = Contributor.objects.get_or_create(
+                login=assignee)
+            assignee_objects_list.append(assignee_object)
+        issue_object.assignees.add(*assignee_objects_list)
+
+        # Saving labels
+        label_objects_list = []
+        for label in labels:
+            label_object, created = Label.objects.get_or_create(name=label)
+            label_objects_list.append(label_object)
+        issue_object.labels.add(*label_objects_list)
+        logger.info('Issue: %s has been saved.' % issue_object)
+    except Exception as ex:
+        logger.error(
+            'Something went wrong saving this issue %s: %s'
+            % (number, ex))

data/management/commands/import_issues_data.py

@@ -0,0 +1,17 @@
+from django.core.management.base import BaseCommand
+
+from data.issues import fetch_issues, import_issue
+
+
+class Command(BaseCommand):
+    help = 'Import issues opened by newcomers'
+
+    def handle(self, *args, **options):
+        github_data = fetch_issues('GitHub')
+        gitlab_data = fetch_issues('GitLab')
+        if github_data:
+            for data in github_data:
+                import_issue('GitHub', data)
+        if gitlab_data:
+            for data in gitlab_data:
+                import_issue('GitLab', data)

data/management/commands/import_merge_requests_data.py

@@ -0,0 +1,17 @@
+from django.core.management.base import BaseCommand
+
+from data.merge_requests import fetch_mrs, import_mr
+
+
+class Command(BaseCommand):
+    help = 'Import mrs opened by newcomers'
+
+    def handle(self, *args, **options):
+        github_data = fetch_mrs('GitHub')
+        gitlab_data = fetch_mrs('GitLab')
+        if github_data:
+            for data in github_data:
+                import_mr('GitHub', data)
+        if gitlab_data:
+            for data in gitlab_data:
+                import_mr('GitLab', data)

data/merge_requests.py

@@ -0,0 +1,101 @@
+import logging
+from dateutil.parser import parse
+import pytz
+
+import requests
+
+from data.models import (
+    MergeRequest,
+    Label,
+    IssueNumber,
+    Contributor,
+    )
+from data.newcomers import active_newcomers
+from data.webservices import webservices_url
+
+
+def fetch_mrs(hoster):
+    """
+    Get mrs opened by newcomers.
+
+    :param hoster: a string representing hoster, e.g. 'GitHub'
+    :return: a json of mrs data
+    """
+    logger = logging.getLogger(__name__)
+    hoster = hoster.lower()
+    import_url = webservices_url('mrs/%s/all' % hoster)
+
+    headers = {'Content-Type': 'application/json'}
+    try:
+        response = requests.get(
+            url=import_url,
+            headers=headers,
+        )
+        response.raise_for_status()
+    except Exception as e:
+        logger.error(e)
+        return
+    mrs = response.json()
+
+    # Removing mrs which are not opened by newcomers
+    mrs_list = []
+    for mr in mrs:
+        if mr['author'] in active_newcomers():
+            mrs_list.append(mr)
+    return mrs_list
+
+
+def import_mr(hoster, mr):
+    """
+    Import mr data to database.
+
+    :param hoster: a string representing hoster
+    :param mr: a dict containing mr's data
+    """
+    logger = logging.getLogger(__name__)
+    number = mr.get('number')
+    assignees = mr.pop('assignees')
+    labels = mr.pop('labels')
+    author = mr.pop('author')
+    repo_id = mr['repo_id']
+    closes_issues = mr.pop('closes_issues')
+
+    # Parse string datetime to datetime object and add timezone support
+    mr['created_at'] = pytz.utc.localize(parse(mr['created_at']))
+    mr['updated_at'] = pytz.utc.localize(parse(mr['updated_at']))
+
+    try:
+        author, created = Contributor.objects.get_or_create(login=author)
+        mr['author'] = author
+        mr['hoster'] = hoster
+        mr_object, created = MergeRequest.objects.get_or_create(**mr)
+
+        # Saving assignees
+        assignee_objects_list = []
+        for assignee in assignees:
+            assignee_object, created = Contributor.objects.get_or_create(
+                login=assignee)
+            assignee_objects_list.append(assignee_object)
+        mr_object.assignees.add(*assignee_objects_list)
+
+        # Saving issues closes by this mr
+        closes_issues_list = []
+        for i_number in closes_issues:
+            issue_number_object, created = (
+                IssueNumber.objects.get_or_create(
+                    number=i_number,
+                    repo_id=repo_id))
+            closes_issues_list.append(issue_number_object)
+        mr_object.closes_issues.add(*closes_issues_list)
+
+        # Saving labels on the mr
+        label_objects_list = []
+        for label in labels:
+            label_object, created = Label.objects.get_or_create(name=label)
+            label_objects_list.append(label_object)
+        mr_object.labels.add(*label_objects_list)
+        logger.info('MR: %s has been saved.' % mr_object)
+    except Exception as ex:
+        logger.error(
+            'Something went wrong saving this mr %s: %s'
+            % (number, ex))

data/migrations/0003_auto_20180801_0456.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2018-08-01 04:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0002_auto_20180704_1130'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Issue',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('number', models.IntegerField()),
+                ('title', models.TextField()),
+                ('repo', models.CharField(default=None, max_length=100, null=True)),
+                ('repo_id', models.IntegerField()),
+                ('created_at', models.DateTimeField()),
+                ('updated_at', models.DateTimeField()),
+                ('state', models.CharField(max_length=100)),
+                ('hoster', models.CharField(max_length=100)),
+                ('assignees', models.ManyToManyField(blank=True, related_name='issue_assignees', to='data.Contributor')),
+                ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='issue_author', to='data.Contributor')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='IssueNumber',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('number', models.IntegerField()),
+                ('repo_id', models.IntegerField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Label',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=300)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='MergeRequest',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('number', models.IntegerField()),
+                ('title', models.TextField()),
+                ('repo_id', models.IntegerField()),
+                ('repo', models.CharField(default=None, max_length=100, null=True)),
+                ('state', models.CharField(max_length=100)),
+                ('created_at', models.DateTimeField()),
+                ('updated_at', models.DateTimeField()),
+                ('ci_status', models.BooleanField()),
+                ('hoster', models.CharField(max_length=100)),
+                ('assignees', models.ManyToManyField(blank=True, related_name='mr_assignees', to='data.Contributor')),
+                ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mr_author', to='data.Contributor')),
+                ('closes_issues', models.ManyToManyField(blank=True, to='data.IssueNumber')),
+                ('labels', models.ManyToManyField(blank=True, to='data.Label')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='issue',
+            name='labels',
+            field=models.ManyToManyField(blank=True, to='data.Label'),
+        ),
+    ]

data/models.py

@@ -22,3 +22,82 @@ def __str__(self):
 
     class Meta:
         ordering = ['login']
+
+
+class Label(models.Model):
+    name = models.CharField(max_length=300)
+
+    def __str__(self):
+        return self.name
+
+
+class Issue(models.Model):
+    number = models.IntegerField()
+    title = models.TextField()
+    repo = models.CharField(max_length=100, default=None, null=True)
+    repo_id = models.IntegerField()
+    author = models.ForeignKey(Contributor,
+                               on_delete=models.CASCADE,
+                               related_name='issue_author')
+    created_at = models.DateTimeField()
+    updated_at = models.DateTimeField()
+    state = models.CharField(max_length=100)
+    labels = models.ManyToManyField(Label, blank=True)
+    assignees = models.ManyToManyField(Contributor,
+                                       related_name='issue_assignees',
+                                       blank=True)
+    hoster = models.CharField(max_length=100)
+
+    def __str__(self):
+        return str(self.title)
+
+
+class IssueNumber(models.Model):
+    number = models.IntegerField()
+    repo_id = models.IntegerField()
+
+    def __str__(self):
+        return str(self.number)
+
+    def get_issue_for_issue_number(self):
+        """
+        Get an issue object which number mathes with the
+        issue number and has same repo_id.
+        """
+        issue = Issue.objects.get(number=self.number,
+                                  repo_id=self.repo_id)
+        return issue
+
+
+class MergeRequest(models.Model):
+    number = models.IntegerField()
+    title = models.TextField()
+    repo_id = models.IntegerField()
+    repo = models.CharField(max_length=100, default=None, null=True)
+    closes_issues = models.ManyToManyField(IssueNumber, blank=True)
+    state = models.CharField(max_length=100)
+    author = models.ForeignKey(Contributor,
+                               on_delete=models.CASCADE,
+                               related_name='mr_author')
+    created_at = models.DateTimeField()
+    updated_at = models.DateTimeField()
+    assignees = models.ManyToManyField(Contributor,
+                                       related_name='mr_assignees',
+                                       blank=True)
+    ci_status = models.BooleanField()
+    labels = models.ManyToManyField(Label, blank=True)
+    hoster = models.CharField(max_length=100)
+
+    def __str__(self):
+        return self.title
+
+    def get_closes_issues_object(self):
+        """
+        Get the list of issues object this mr is closing.
+        """
+        issues_object_list = []
+        issue_numbers = self.closes_issues.all()
+        for issue_number in issue_numbers:
+            issue_object = issue_number.get_issue_for_issue_number()
+            issues_object_list.append(issue_object)
+        return issues_object_list