Start repo validation on activation (#236)

* factored out repo.create_and_start_job method

* validate repo on activation

goodtablesio/integrations/github/blueprint.py

@@ -1,28 +1,19 @@
-import uuid
 import logging
-
-from celery import chain
 from flask import Blueprint, request, abort, jsonify
 from flask_login import login_required, current_user
-
 from goodtablesio import settings
 from goodtablesio.services import database
-from goodtablesio.models.job import Job
 from goodtablesio.models.internal_job import InternalJob
-from goodtablesio.tasks.validate import validate
 from goodtablesio.utils.signature import validate_signature
 from goodtablesio.integrations.github.models.repo import GithubRepo
-from goodtablesio.integrations.github.tasks.jobconf import get_validation_conf
 from goodtablesio.integrations.github.tasks.repos import sync_user_repos
-from goodtablesio.integrations.github.utils.status import set_commit_status
 from goodtablesio.integrations.github.utils.hook import (
-    activate_hook, deactivate_hook, get_details_from_hook_payload,
-    get_tokens_for_job)
-
-
+    activate_hook, deactivate_hook, get_details_from_hook_payload)
 log = logging.getLogger(__name__)
 
 
+# Module API
+
 github = Blueprint('github', __name__, url_prefix='/github')
 
 
@@ -34,7 +25,7 @@ def record_params(setup_state):
 @github.route('/hook', methods=['POST'])
 def create_job():
 
-    # Validate signature
+    # Validate signature (throws 400 on invalid)
     if not github.debug:
         key = settings.GITHUB_HOOK_SECRET
         text = request.data
@@ -44,66 +35,35 @@ def create_job():
             log.error(msg)
             abort(400, msg)
 
-    # Get payload parameters (throws 400 if no data or bad JSON)
+    # Get payload details (throws 400 if no data or bad JSON)
     payload = request.get_json()
-
     details = get_details_from_hook_payload(payload)
     if details is None:
         msg = 'Wrong payload received'
         log.error(msg)
         abort(400, msg)
     if details == {}:
-        return 'No job triggered'
+        return jsonify({})
 
-    owner = details['owner']
-    repo = details['repo']
-    sha = details['sha']
-
-    # Check repo exists
+    # Get source (throw 400 if no source)
+    source_owner = details['owner']
+    source_repo = details['repo']
     if details['is_pr']:
         source_owner = details['base_owner']
         source_repo = details['base_repo']
-    else:
-        source_owner = owner
-        source_repo = repo
-
     source = database['session'].query(GithubRepo).filter(
-        GithubRepo.name == '{0}/{1}'.format(
-            source_owner, source_repo)).one_or_none()
-
+        GithubRepo.name == '%s/%s' % (source_owner, source_repo)).one_or_none()
     if not source:
         msg = 'A job was requested on a repository not present in the DB'
         log.error(msg)
         abort(400, msg)
 
-    # Save job to database
-    job_id = str(uuid.uuid4())
-    params = {
-        'id': job_id,
-        'integration_name': 'github',
-        'source_id': source.id,
-        'conf': details
-    }
-    job = Job(**params)
-    job.source = source
-
-    database['session'].add(job)
-    database['session'].commit()
-
-    tokens = get_tokens_for_job(job)
-
-    # Set GitHub status
-    set_commit_status(
-        'pending',
-        owner=owner,
-        repo=repo,
-        sha=sha,
-        job_number=job.number,
-        is_pr=details['is_pr'],
-        tokens=tokens)
-
-    # Run validation
-    _run_validation(owner, repo, sha, job_id)
+    # Create and start job (throw 400 if not started)
+    job_id = source.create_and_start_job(conf=details)
+    if not job_id:
+        msg = 'A job was requested but can\'t be started'
+        log.error(msg)
+        abort(400, msg)
 
     return jsonify({'job_id': job_id})
 
@@ -215,6 +175,10 @@ def api_repo_activate(repo_id):
             error = 'Repo activation error'
             log.exception(exception)
 
+    # Validate repo
+    if not error:
+        repo.create_and_start_job()
+
     return jsonify({
         'error': error,
     })
@@ -266,10 +230,3 @@ def _is_user_repos_syncing(user_id):
             user_id=user_id,
             finished=None).
         count())
-
-
-def _run_validation(owner, repo, sha, job_id):
-    tasks_chain = chain(
-        get_validation_conf.s(owner, repo, sha, job_id=job_id),
-        validate.s(job_id=job_id))
-    tasks_chain.delay()

goodtablesio/integrations/github/models/repo.py

@@ -1,6 +1,16 @@
+import uuid
+import logging
+from goodtablesio.services import database
+from goodtablesio.models.job import Job
 from goodtablesio.models.source import Source
+from goodtablesio.integrations.github.utils.status import set_commit_status
+from goodtablesio.integrations.github.utils.validation import run_validation
+from goodtablesio.integrations.github.utils.repos import get_default_repo_details
+log = logging.getLogger(__name__)
 
 
+# Module API
+
 class GithubRepo(Source):
 
     __mapper_args__ = {
@@ -9,22 +19,64 @@ class GithubRepo(Source):
 
     @property
     def url(self):
-
         parts = self.name.split('/')
-
         template = 'https://github.com/{owner}/{repo}'
         return template.format(owner=parts[0], repo=parts[1])
 
     @property
     def owner(self):
-
         parts = self.name.split('/')
-
         return parts[0]
 
     @property
     def repo(self):
-
         parts = self.name.split('/')
-
         return parts[1]
+
+    @property
+    def tokens(self):
+        return [user.github_oauth_token for user in self.users
+            if user.github_oauth_token]
+
+    def create_and_start_job(self, conf=None):
+
+        # Get tokens
+        if not self.tokens:
+            log.error('No GitHub tokens available to perform the job')
+            return None
+
+        # Get default repo details
+        if not conf:
+            conf = get_default_repo_details(self.owner, self.repo, token=self.tokens[0])
+            if not conf:
+                log.error('No default repo details are available')
+                return None
+
+        # Save job to database
+        job_id = str(uuid.uuid4())
+        params = {
+            'id': job_id,
+            'integration_name': 'github',
+            'source_id': self.id,
+            'conf': conf
+        }
+        job = Job(**params)
+        job.source = self
+        database['session'].add(job)
+        database['session'].commit()
+
+        # Set GitHub status
+        set_commit_status(
+            'pending',
+            owner=conf['owner'],
+            repo=conf['repo'],
+            sha=conf['sha'],
+            job_number=job.number,
+            is_pr=conf['is_pr'],
+            tokens=self.tokens)
+
+        # Run validation
+        run_validation(conf['owner'], conf['repo'], conf['sha'],
+            job_id=job_id, tokens=self.tokens)
+
+        return job_id

goodtablesio/integrations/github/models/user.py

@@ -1,9 +1,10 @@
 import cryptography
-
 from goodtablesio.crypto import encrypt_string, decrypt_string
 from goodtablesio.models.user import User
 
 
+# Helpers
+
 def get_token(self):
     token = self.conf.get('github_oauth_token')
     if token:
@@ -27,7 +28,7 @@ def del_token(self):
     del self.conf['github_oauth_token']
 
 
-github_oauth_token = property(
-    get_token, set_token, del_token, 'GitHub OAuth Token')
+# Module API
 
-setattr(User, 'github_oauth_token', github_oauth_token)
+setattr(User, 'github_oauth_token', property(
+    get_token, set_token, del_token, 'GitHub OAuth Token'))

goodtablesio/integrations/github/signals.py

@@ -1,12 +1,12 @@
 from celery import signals
-
 from goodtablesio.services import database
 from goodtablesio.models.job import Job
 from goodtablesio.tasks.validate import validate
 from goodtablesio.integrations.github.utils.status import set_commit_status
-from goodtablesio.integrations.github.utils.hook import get_tokens_for_job
 
 
+# Module API
+
 @signals.task_postrun.connect(sender=validate)
 def post_task_handler(**kwargs):
 
@@ -25,13 +25,11 @@ def post_task_handler(**kwargs):
         else:
             github_status = 'error'
 
-        tokens = get_tokens_for_job(job)
-
         set_commit_status(
            github_status,
            owner=job.conf['owner'],
            repo=job.conf['repo'],
            sha=job.conf['sha'],
            job_number=job.number,
            is_pr=job.conf['is_pr'],
-           tokens=tokens)
+           tokens=job.source.tokens)

goodtablesio/integrations/github/tasks/jobconf.py

@@ -1,21 +1,18 @@
 import logging
-
 import requests
 from github3 import GitHub
-
 from goodtablesio.services import database
 from goodtablesio.models.job import Job
 from goodtablesio.celery_app import celery_app
 from goodtablesio.tasks.base import JobTask
 from goodtablesio.utils.jobconf import make_validation_conf
-from goodtablesio.integrations.github.utils.hook import get_tokens_for_job
-
-
 log = logging.getLogger(__name__)
 
 
+# Module API
+
 @celery_app.task(name='goodtablesio.github.get_validation_conf', base=JobTask)
-def get_validation_conf(owner, repo, sha, job_id):
+def get_validation_conf(owner, repo, sha, job_id, tokens):
     """Celery tast to get validation conf.
     """
 
@@ -25,11 +22,6 @@ def get_validation_conf(owner, repo, sha, job_id):
     database['session'].add(job)
     database['session'].commit()
 
-    tokens = get_tokens_for_job(job)
-    if not tokens:
-        log.error('No GitHub tokens available to perform the job')
-        return {}
-
     # Get validation conf
     job_base = _get_job_base(owner, repo, sha)
     job_files = _get_job_files(owner, repo, sha, tokens)
@@ -41,7 +33,6 @@ def get_validation_conf(owner, repo, sha, job_id):
 
 # Internal
 
-
 def _get_job_base(owner, repo, sha):
     """Get job's base url.
     """

goodtablesio/integrations/github/tasks/repos.py

@@ -9,6 +9,8 @@
 from goodtablesio.integrations.github.utils.repos import iter_repos_by_token
 
 
+# Module API
+
 @celery_app.task(name='goodtablesio.github.sync_user_repos',
                  queue='internal', base=InternalJobTask)
 def sync_user_repos(user_id, job_id):
@@ -68,6 +70,8 @@ def sync_user_repos(user_id, job_id):
     database['session'].commit()
 
 
+# Internal
+
 def _can_see_private_repos(user, repo_data):
     # TODO: check that there is any subscription that gives access to this
     # GitHub organization

goodtablesio/integrations/github/utils/hook.py

@@ -72,8 +72,3 @@ def get_details_from_hook_payload(payload):
         return None
 
     return details
-
-
-def get_tokens_for_job(job):
-    return [user.github_oauth_token
-            for user in job.source.users if user.github_oauth_token]

goodtablesio/integrations/github/utils/repos.py

@@ -1,4 +1,6 @@
+import logging
 from github3 import GitHub
+log = logging.getLogger(__name__)
 
 
 # Module API
@@ -22,3 +24,29 @@ def iter_repos_by_token(token):
             },
             'active': active
         }
+
+
+def get_default_repo_details(owner, repo, token):
+    """Return defaults repo details.
+    """
+    try:
+        client = GitHub(token=token)
+        repo_client = client.repository(owner, repo)
+        branch_client = repo_client.branch(repo_client.default_branch)
+        branch_data = branch_client.to_json()
+        branch_name = branch_data['name']
+        author_name = branch_data['commit']['author']['login']
+        commit_message = branch_data['commit']['commit']['message']
+        sha = branch_data['commit']['sha']
+    except Exception as exception:
+        log.exception(exception)
+        return None
+    return {
+        'is_pr': False,
+        'owner': owner,
+        'repo': repo,
+        'sha': sha,
+        'branch_name': branch_name,
+        'author_name': author_name,
+        'commit_message': commit_message,
+    }