diff --git a/.ci/build.sh b/.ci/build.sh index de6f5d00..5114a99f 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -24,6 +24,8 @@ python manage.py fetch_deployed_data _site $ISSUES_JSON --repo-name gh-board python manage.py migrate python manage.py test python manage.py import_contributors_data +python manage.py import_issues_data +python manage.py import_merge_requests_data python manage.py import_openhub_data if [[ -f "_site/$META_REVIEW_DATA" ]]; then diff --git a/data/issues.py b/data/issues.py new file mode 100644 index 00000000..affe74b8 --- /dev/null +++ b/data/issues.py @@ -0,0 +1,88 @@ +import logging +from dateutil.parser import parse +import pytz + +import requests + +from data.models import ( + Issue, + Label, + Contributor, + ) +from data.newcomers import active_newcomers +from data.webservices import webservices_url + + +def fetch_issues(hoster): + """ + Get issues opened by newcomers. + + :param hoster: a string representing hoster, e.g. 'GitHub' + :return: a json of issues data + """ + logger = logging.getLogger(__name__) + hoster = hoster.lower() + import_url = webservices_url('issues/%s/all' % hoster) + + headers = {'Content-Type': 'application/json'} + try: + response = requests.get( + url=import_url, + headers=headers, + ) + response.raise_for_status() + except Exception as e: + logger.error(e) + return + issues = response.json() + + # Removing issues which are not opened by newcomers + issues_list = [] + for issue in issues: + if issue['author'] in active_newcomers(): + issues_list.append(issue) + return issues_list + + +def import_issue(hoster, issue): + """ + Import issue data to database. + + :param hoster: a string representing hoster + :param issue: a dict containing issue's data + """ + logger = logging.getLogger(__name__) + number = issue.get('number') + assignees = issue.pop('assignees') + labels = issue.pop('labels') + author = issue.pop('author') + + # Parse string datetime to datetime object and add timezone support + issue['created_at'] = pytz.utc.localize(parse(issue['created_at'])) + issue['updated_at'] = pytz.utc.localize(parse(issue['updated_at'])) + + try: + author, created = Contributor.objects.get_or_create(login=author) + issue['author'] = author + issue['hoster'] = hoster + issue_object, created = Issue.objects.get_or_create(**issue) + + # Saving assignees + assignee_objects_list = [] + for assignee in assignees: + assignee_object, created = Contributor.objects.get_or_create( + login=assignee) + assignee_objects_list.append(assignee_object) + issue_object.assignees.add(*assignee_objects_list) + + # Saving labels + label_objects_list = [] + for label in labels: + label_object, created = Label.objects.get_or_create(name=label) + label_objects_list.append(label_object) + issue_object.labels.add(*label_objects_list) + logger.info('Issue: %s has been saved.' % issue_object) + except Exception as ex: + logger.error( + 'Something went wrong saving this issue %s: %s' + % (number, ex)) diff --git a/data/management/commands/import_issues_data.py b/data/management/commands/import_issues_data.py new file mode 100644 index 00000000..59e68cd5 --- /dev/null +++ b/data/management/commands/import_issues_data.py @@ -0,0 +1,17 @@ +from django.core.management.base import BaseCommand + +from data.issues import fetch_issues, import_issue + + +class Command(BaseCommand): + help = 'Import issues opened by newcomers' + + def handle(self, *args, **options): + github_data = fetch_issues('GitHub') + gitlab_data = fetch_issues('GitLab') + if github_data: + for data in github_data: + import_issue('GitHub', data) + if gitlab_data: + for data in gitlab_data: + import_issue('GitLab', data) diff --git a/data/management/commands/import_merge_requests_data.py b/data/management/commands/import_merge_requests_data.py new file mode 100644 index 00000000..ecc83706 --- /dev/null +++ b/data/management/commands/import_merge_requests_data.py @@ -0,0 +1,17 @@ +from django.core.management.base import BaseCommand + +from data.merge_requests import fetch_mrs, import_mr + + +class Command(BaseCommand): + help = 'Import mrs opened by newcomers' + + def handle(self, *args, **options): + github_data = fetch_mrs('GitHub') + gitlab_data = fetch_mrs('GitLab') + if github_data: + for data in github_data: + import_mr('GitHub', data) + if gitlab_data: + for data in gitlab_data: + import_mr('GitLab', data) diff --git a/data/merge_requests.py b/data/merge_requests.py new file mode 100644 index 00000000..e7c005d3 --- /dev/null +++ b/data/merge_requests.py @@ -0,0 +1,101 @@ +import logging +from dateutil.parser import parse +import pytz + +import requests + +from data.models import ( + MergeRequest, + Label, + IssueNumber, + Contributor, + ) +from data.newcomers import active_newcomers +from data.webservices import webservices_url + + +def fetch_mrs(hoster): + """ + Get mrs opened by newcomers. + + :param hoster: a string representing hoster, e.g. 'GitHub' + :return: a json of mrs data + """ + logger = logging.getLogger(__name__) + hoster = hoster.lower() + import_url = webservices_url('mrs/%s/all' % hoster) + + headers = {'Content-Type': 'application/json'} + try: + response = requests.get( + url=import_url, + headers=headers, + ) + response.raise_for_status() + except Exception as e: + logger.error(e) + return + mrs = response.json() + + # Removing mrs which are not opened by newcomers + mrs_list = [] + for mr in mrs: + if mr['author'] in active_newcomers(): + mrs_list.append(mr) + return mrs_list + + +def import_mr(hoster, mr): + """ + Import mr data to database. + + :param hoster: a string representing hoster + :param mr: a dict containing mr's data + """ + logger = logging.getLogger(__name__) + number = mr.get('number') + assignees = mr.pop('assignees') + labels = mr.pop('labels') + author = mr.pop('author') + repo_id = mr['repo_id'] + closes_issues = mr.pop('closes_issues') + + # Parse string datetime to datetime object and add timezone support + mr['created_at'] = pytz.utc.localize(parse(mr['created_at'])) + mr['updated_at'] = pytz.utc.localize(parse(mr['updated_at'])) + + try: + author, created = Contributor.objects.get_or_create(login=author) + mr['author'] = author + mr['hoster'] = hoster + mr_object, created = MergeRequest.objects.get_or_create(**mr) + + # Saving assignees + assignee_objects_list = [] + for assignee in assignees: + assignee_object, created = Contributor.objects.get_or_create( + login=assignee) + assignee_objects_list.append(assignee_object) + mr_object.assignees.add(*assignee_objects_list) + + # Saving issues closes by this mr + closes_issues_list = [] + for i_number in closes_issues: + issue_number_object, created = ( + IssueNumber.objects.get_or_create( + number=i_number, + repo_id=repo_id)) + closes_issues_list.append(issue_number_object) + mr_object.closes_issues.add(*closes_issues_list) + + # Saving labels on the mr + label_objects_list = [] + for label in labels: + label_object, created = Label.objects.get_or_create(name=label) + label_objects_list.append(label_object) + mr_object.labels.add(*label_objects_list) + logger.info('MR: %s has been saved.' % mr_object) + except Exception as ex: + logger.error( + 'Something went wrong saving this mr %s: %s' + % (number, ex)) diff --git a/data/migrations/0003_auto_20180801_0456.py b/data/migrations/0003_auto_20180801_0456.py new file mode 100644 index 00000000..3b5b7662 --- /dev/null +++ b/data/migrations/0003_auto_20180801_0456.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.11 on 2018-08-01 04:56 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0002_auto_20180704_1130'), + ] + + operations = [ + migrations.CreateModel( + name='Issue', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('number', models.IntegerField()), + ('title', models.TextField()), + ('repo', models.CharField(default=None, max_length=100, null=True)), + ('repo_id', models.IntegerField()), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('state', models.CharField(max_length=100)), + ('hoster', models.CharField(max_length=100)), + ('assignees', models.ManyToManyField(blank=True, related_name='issue_assignees', to='data.Contributor')), + ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='issue_author', to='data.Contributor')), + ], + ), + migrations.CreateModel( + name='IssueNumber', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('number', models.IntegerField()), + ('repo_id', models.IntegerField()), + ], + ), + migrations.CreateModel( + name='Label', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=300)), + ], + ), + migrations.CreateModel( + name='MergeRequest', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('number', models.IntegerField()), + ('title', models.TextField()), + ('repo_id', models.IntegerField()), + ('repo', models.CharField(default=None, max_length=100, null=True)), + ('state', models.CharField(max_length=100)), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('ci_status', models.BooleanField()), + ('hoster', models.CharField(max_length=100)), + ('assignees', models.ManyToManyField(blank=True, related_name='mr_assignees', to='data.Contributor')), + ('author', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mr_author', to='data.Contributor')), + ('closes_issues', models.ManyToManyField(blank=True, to='data.IssueNumber')), + ('labels', models.ManyToManyField(blank=True, to='data.Label')), + ], + ), + migrations.AddField( + model_name='issue', + name='labels', + field=models.ManyToManyField(blank=True, to='data.Label'), + ), + ] diff --git a/data/models.py b/data/models.py index e390e2a5..6c8ffcac 100644 --- a/data/models.py +++ b/data/models.py @@ -22,3 +22,82 @@ def __str__(self): class Meta: ordering = ['login'] + + +class Label(models.Model): + name = models.CharField(max_length=300) + + def __str__(self): + return self.name + + +class Issue(models.Model): + number = models.IntegerField() + title = models.TextField() + repo = models.CharField(max_length=100, default=None, null=True) + repo_id = models.IntegerField() + author = models.ForeignKey(Contributor, + on_delete=models.CASCADE, + related_name='issue_author') + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + state = models.CharField(max_length=100) + labels = models.ManyToManyField(Label, blank=True) + assignees = models.ManyToManyField(Contributor, + related_name='issue_assignees', + blank=True) + hoster = models.CharField(max_length=100) + + def __str__(self): + return str(self.title) + + +class IssueNumber(models.Model): + number = models.IntegerField() + repo_id = models.IntegerField() + + def __str__(self): + return str(self.number) + + def get_issue(self): + """ + Get an issue object which number mathes with the + issue number and has same repo_id. + """ + issue = Issue.objects.get(number=self.number, + repo_id=self.repo_id) + return issue + + +class MergeRequest(models.Model): + number = models.IntegerField() + title = models.TextField() + repo_id = models.IntegerField() + repo = models.CharField(max_length=100, default=None, null=True) + closes_issues = models.ManyToManyField(IssueNumber, blank=True) + state = models.CharField(max_length=100) + author = models.ForeignKey(Contributor, + on_delete=models.CASCADE, + related_name='mr_author') + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + assignees = models.ManyToManyField(Contributor, + related_name='mr_assignees', + blank=True) + ci_status = models.BooleanField() + labels = models.ManyToManyField(Label, blank=True) + hoster = models.CharField(max_length=100) + + def __str__(self): + return self.title + + def get_closes_issues_object(self): + """ + Get the list of issues object this mr is closing. + """ + issues_object_list = [] + issue_numbers = self.closes_issues.all() + for issue_number in issue_numbers: + issue_object = issue_number.get_issue() + issues_object_list.append(issue_object) + return issues_object_list diff --git a/data/newcomers.py b/data/newcomers.py new file mode 100644 index 00000000..865549a8 --- /dev/null +++ b/data/newcomers.py @@ -0,0 +1,21 @@ +from functools import lru_cache + +import requests + +from data.webservices import webservices_url + + +@lru_cache(maxsize=32) +def active_newcomers(): + """ + Get the list of newcomers active in the last three months. + + :return: the list of newcomer usernames + """ + NEWCOMERS_URL = webservices_url('newcomers/active') + response = requests.get(NEWCOMERS_URL) + newcomers = response.json() + active_newcomers_list = [] + for newcomer in newcomers: + active_newcomers_list.append(newcomer['username']) + return active_newcomers_list diff --git a/data/tests/test_management_commands.py b/data/tests/test_management_commands.py index 07c855ff..f1309700 100644 --- a/data/tests/test_management_commands.py +++ b/data/tests/test_management_commands.py @@ -3,7 +3,7 @@ from django.core.management import call_command from django.test import TestCase -from data.models import Contributor +from data.models import Contributor, Issue, MergeRequest class ImportContributorDataTest(TestCase): @@ -19,3 +19,33 @@ def test_command_import_contributors_data(self): 'No record of contributors from webservices') self.assertIn('jayvdb', [contributor.login for contributor in contributors]) + + +class ImportIssuesDataTest(TestCase): + + @classmethod + def setUpTestData(cls): + call_command('import_issues_data') + + def test_command_import_issues_data(self): + issues = Issue.objects.all() + if not issues: + raise unittest.SkipTest( + 'No record of issues from webservices') + self.assertIn('testuser', + [issue.author.login for issue in issues]) + + +class ImportMergeRequestDataTest(TestCase): + + @classmethod + def setUpTestData(cls): + call_command('import_merge_requests_data') + + def test_command_import_issues_data(self): + mrs = MergeRequest.objects.all() + if not mrs: + raise unittest.SkipTest( + 'No record of mrs from webservices') + self.assertIn('testuser', + [mr.author.login for mr in mrs]) diff --git a/data/tests/test_models.py b/data/tests/test_models.py index b0e246ef..ce706ded 100644 --- a/data/tests/test_models.py +++ b/data/tests/test_models.py @@ -1,6 +1,16 @@ +from dateutil.parser import parse +import pytz + from django.test import TestCase -from data.models import Contributor +from data.models import ( + Contributor, + Label, + IssueNumber, + Issue, + MergeRequest, + ) +from community.git import get_org_name class ContributorModelTest(TestCase): @@ -37,3 +47,239 @@ def test_class_meta_ordering(self): contributors = Contributor.objects.all() self.assertEquals(contributors[0].login, 'sks444') self.assertEquals(contributors[1].login, 'test') + + +class LabelModelTest(TestCase): + + @classmethod + def setUpTestData(cls): + # Set up non-modified objects used by all methods + Label.objects.create(name='difficulty/newcomer') + + def test_field_label(self): + label = Label.objects.get(id=1) + name = label._meta.get_field('name').verbose_name + self.assertEquals(name, 'name') + + def test_object_name_is_name(self): + label = Label.objects.get(id=1) + expected_object_name = 'difficulty/newcomer' + self.assertEquals(expected_object_name, str(label)) + + +class IssueModelTest(TestCase): + + @classmethod + def setUpTestData(cls): + # Set up non-modified objects used by all methods + contributor = Contributor.objects.create(login='sks444', + name='Shrikrishna Singh') + label = Label.objects.create(name='difficulty/newcomer') + created_at = pytz.utc.localize(parse('2017-11-07T08:01:21')) + updated_at = pytz.utc.localize(parse('2018-03-18T00:00:11')) + issue = Issue.objects.create(number=1, + title='Test issue', + repo_id=1, + author=contributor, + created_at=created_at, + updated_at=updated_at, + state='open', + hoster='GitHub') + issue.labels.add(label) + issue.assignees.add(contributor) + + def test_field_label(self): + issue = Issue.objects.get(number=1) + number = issue._meta.get_field('number').verbose_name + title = issue._meta.get_field('title').verbose_name + repo_id = issue._meta.get_field('repo_id').verbose_name + repo = issue._meta.get_field('repo').verbose_name + author = issue._meta.get_field('author').verbose_name + created_at = issue._meta.get_field( + 'created_at').verbose_name + updated_at = issue._meta.get_field( + 'updated_at').verbose_name + state = issue._meta.get_field('state').verbose_name + hoster = issue._meta.get_field('hoster').verbose_name + labels = issue._meta.get_field('labels').verbose_name + assignees = issue._meta.get_field( + 'assignees').verbose_name + self.assertEquals(number, 'number') + self.assertEquals(title, 'title') + self.assertEquals(repo_id, 'repo id') + self.assertEquals(repo, 'repo') + self.assertEquals(author, 'author') + self.assertEquals(created_at, 'created at') + self.assertEquals(updated_at, 'updated at') + self.assertEquals(state, 'state') + self.assertEquals(hoster, 'hoster') + self.assertEquals(labels, 'labels') + self.assertEquals(assignees, 'assignees') + + def test_object_name_is_title(self): + issue = Issue.objects.get(number=1) + expected_object_name = 'Test issue' + self.assertEquals(expected_object_name, str(issue)) + + def test_many_to_many_field(self): + issue = Issue.objects.get(number=1) + label = Label.objects.get(id=1) + assignee = Contributor.objects.get(login='sks444') + + # Test issue has many to many field with label + self.assertEquals(issue.labels.get(pk=label.pk), + label) + + # Test issue has many to many field with contributor + self.assertEquals(issue.assignees.get(pk=assignee.pk), + assignee) + + +class IssueNumberModelTest(TestCase): + + @classmethod + def setUpTestData(cls): + # Set up non-modified objects used by all methods + contributor = Contributor.objects.create(login='Abhisek-') + created_at = pytz.utc.localize(parse('2016-12-13 17:45:38')) + updated_at = pytz.utc.localize(parse('2017-12-21 00:01:23')) + org_name = get_org_name() + repo = org_name + '/' + org_name + '-quickstart' + Issue.objects.create(number=57, + repo=repo, + title='Remove the python 3.3.6 dependency', + repo_id=52889504, + author=contributor, + created_at=created_at, + updated_at=updated_at, + state='closed', + hoster='GitHub') + IssueNumber.objects.create(number=57, repo_id=52889504) + + def test_field_label(self): + issue_number = IssueNumber.objects.get(number=57) + number = issue_number._meta.get_field('number').verbose_name + repo_id = issue_number._meta.get_field('repo_id').verbose_name + self.assertEquals(number, 'number') + self.assertEquals(repo_id, 'repo id') + + def test_object_name_is_number(self): + issue_number = IssueNumber.objects.get(number=57) + expected_object_name = '57' + self.assertEquals(expected_object_name, str(issue_number)) + + def test_get_issue(self): + issue_number = IssueNumber.objects.get(number=57, + repo_id=52889504) + issue = Issue.objects.get(number=57, repo_id=52889504) + + # Get the issue object from the method + issue_object = issue_number.get_issue() + + # Both object should be equal + self.assertEquals(issue, issue_object) + + +class MergeRequestModelTest(): + + @classmethod + def setUpTestData(cls): + # Set up non-modified objects used by all methods + contributor = Contributor.objects.create(login='sks444', + name='Shrikrishna Singh') + label = Label.objects.create(name='status/STALE') + created_at = pytz.utc.localize(parse('2016-12-13 18:11:57')) + updated_at = pytz.utc.localize(parse('2016-12-20 06:48:23')) + org_name = get_org_name() + repo = org_name + '/' + org_name + '-quickstart' + mr = MergeRequest.objects.create(number=58, + repo=repo, + title='Removed python 3.3 dependency', + repo_id=52889504, + author=contributor, + created_at=created_at, + updated_at=updated_at, + state='merged', + hoster='GitHub', + ci_status=1) + issue = IssueNumber.objects.create(number=57, repo_id=52889504) + mr.closes_issues.add(issue) + mr.labels.add(label) + mr.assignees.add(contributor) + Issue.objects.create(number=57, + repo=repo, + title='Remove the python 3.3.6 dependency', + repo_id=52889504, + author=contributor, + created_at=created_at, + updated_at=updated_at, + state='closed', + hoster='GitHub') + + def test_field_label(self): + mr = MergeRequest.objects.get(number=58, repo_id=52889504) + number = mr._meta.get_field('number').verbose_name + title = mr._meta.get_field('title').verbose_name + repo_id = mr._meta.get_field('repo_id').verbose_name + repo = mr._meta.get_field('repo').verbose_name + author = mr._meta.get_field('author').verbose_name + created_at = mr._meta.get_field( + 'created_at').verbose_name + updated_at = mr._meta.get_field( + 'updated_at').verbose_name + state = mr._meta.get_field('state').verbose_name + hoster = mr._meta.get_field('hoster').verbose_name + ci_status = mr._meta.get_field('ci_status').verbose_name + labels = mr._meta.get_field('labels').verbose_name + assignees = mr._meta.get_field( + 'assignees').verbose_name + closes_issues = mr._meta.get_field( + 'closes_issues').verbose_name + self.assertEquals(number, 'number') + self.assertEquals(title, 'title') + self.assertEquals(repo_id, 'repo id') + self.assertEquals(repo, 'repo') + self.assertEquals(author, 'author') + self.assertEquals(created_at, 'created at') + self.assertEquals(updated_at, 'updated at') + self.assertEquals(state, 'state') + self.assertEquals(hoster, 'hoster') + self.assertEquals(ci_status, 'ci status') + self.assertEquals(labels, 'labels') + self.assertEquals(assignees, 'assignees') + self.assertEquals(closes_issues, 'closes issues') + + def test_object_name_is_title(self): + mr = MergeRequest.objects.get(number=58, repo_id=52889504) + expected_object_name = 'Test Merge Request' + self.assertEquals(expected_object_name, str(mr)) + + def test_many_to_many_field(self): + mr = MergeRequest.objects.get(number=58, repo_id=52889504) + label = Label.objects.get(id=1) + assignee = Contributor.objects.get(login='sks444') + closes_issue = IssueNumber.objects.get(number=57, repo_id=52889504) + + # Test mr has many to many field with Label + self.assertEquals(mr.labels.get(pk=label.pk), + label) + + # Test mr has many to many field with Contributor + self.assertEquals(mr.assignees.get(pk=assignee.pk), + assignee) + + # Test mr has many to many field with IssueNumber + self.assertEquals(mr.closes_issues.get(pk=closes_issue.pk), + closes_issue) + + def test_get_closes_issue_objects(self): + mr = MergeRequest.objects.get(number=58, repo_id=52889504) + closes_issues = mr.test_get_closes_issue_objects() + + # Expected closes issue objects list + issue_objects_list = [ + Issue.objects.get(number=57, repo_id=52889504), + ] + + # Both the issue objects list should be equal + self.assertEquals(closes_issues, issue_objects_list)