Merge pull request #1 from CIMAC-CIDC/deploy-dummy-function

Create + deploy function triggered by upload event

.env.prod.yaml

@@ -1,6 +1,6 @@
 CLOUD_SQL_INSTANCE_NAME: 'cidc-dfci:us-central1:cidc-postgres'
 CLOUD_SQL_DB_USER: 'postgres'
 CLOUD_SQL_DB_NAME: 'cidc'
-GOOGLE_PROJECT_ID: 'cidc-dfci'
+GOOGLE_SECRETS_BUCKET: 'cidc-secrets-prod'
 GOOGLE_UPLOAD_BUCKET: 'cidc-uploads-prod'
 GOOGLE_UPLOAD_TOPIC: 'uploads'

.env.staging.yaml

@@ -1,6 +1,6 @@
 CLOUD_SQL_INSTANCE_NAME: 'cidc-dfci-staging:us-central1:cidc-postgres'
 CLOUD_SQL_DB_USER: 'postgres'
 CLOUD_SQL_DB_NAME: 'cidc'
-GOOGLE_PROJECT_ID: 'cidc-dfci-staging'
+GOOGLE_SECRETS_BUCKET: 'cidc-secrets-staging'
 GOOGLE_UPLOAD_BUCKET: 'cidc-uploads-staging'
 GOOGLE_UPLOAD_TOPIC: 'uploads'

.gcloudignore

@@ -0,0 +1,17 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules
+#!include:.gitignore

.travis.yml

@@ -40,10 +40,10 @@ script:
 
 deploy:
   - provider: script
-    script: echo "Deploy cloud functions to staging here..."
+    script: bash deploy/pubsub.sh ingest_upload uploads .env.staging.yaml
     on:
       branch: master
   - provider: script
-    script: echo "Deploy cloud functions to production here..."
+    script: bash deploy/pubsub.sh ingest_upload uploads .env.prod.yaml
     on:
       branch: production

deploy/pubsub.sh

@@ -0,0 +1,13 @@
+# !/usr/bin/bash
+# Deploy a cloud function to a topic
+
+ENTRYPOINT=$1
+TOPIC=$2
+ENVFILE=$3
+PROJECT=$(gcloud config get-value project)
+
+gcloud functions deploy $ENTRYPOINT \
+    --runtime python37 \
+    --trigger-topic $TOPIC \
+    --env-vars-file $ENVFILE \
+    --project $PROJECT

functions/__init__.py

@@ -1,3 +1,3 @@
-"""Entrypoint for CIDC cloud functions."""
+"""The CIDC cloud functions."""
 
-from functions import *
+from .uploads import ingest_upload

functions/settings.py

@@ -0,0 +1,18 @@
+"""Configuration for CIDC functions."""
+import os
+
+from cidc_api.config import get_sqlalchemy_database_uri
+
+GOOGLE_CLOUD_PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT")
+
+if not GOOGLE_CLOUD_PROJECT:
+    from dotenv import load_dotenv
+
+    # We're running locally, so load config from .env
+    load_dotenv()
+    GOOGLE_CLOUD_PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT")
+
+
+SQLALCHEMY_DATABASE_URI = get_sqlalchemy_database_uri()
+GOOGLE_UPLOAD_BUCKET = os.environ.get("GOOGLE_UPLOAD_BUCKET")
+GOOGLE_UPLOAD_TOPIC = os.environ.get("GOOGLE_UPLOAD_TOPIC")

functions/uploads.py

@@ -0,0 +1,17 @@
+"""A pub/sub triggered functions that respond to data upload events"""
+import base64
+
+from .util import BackgroundContext, extract_pubsub_data
+
+
+def ingest_upload(event: dict, context: BackgroundContext):
+    """
+    When a successful upload event is published, move the data associated
+    with the upload job into the download bucket and merge the upload metadata
+    into the appropriate clinical trial JSON.
+
+    TODO: actually implement the above functionality. Right now, the function
+    just logs the ID of the upload job.
+    """
+    job_id = extract_pubsub_data(event)
+    print(f"Received upload success event for Job {job_id}")

functions/util.py

@@ -0,0 +1,24 @@
+"""Helpers for working with Cloud Functions."""
+import base64
+from typing import NamedTuple
+
+
+def extract_pubsub_data(event: dict):
+    """Pull out and decode data from a pub/sub event."""
+    # Pub/sub event data is base64-encoded
+    b64data = event["data"]
+    data = base64.b64decode(b64data).decode("utf-8")
+    return data
+
+
+class BackgroundContext(NamedTuple):
+    """
+    Model of the context object passed to a background cloud function.
+    
+    Based on: https://cloud.google.com/functions/docs/writing/background#function_parameters
+    """
+
+    event_id: str
+    timestamp: str  # ISO 8601
+    event_type: str  # e.g., "google.pubsub.topic.publish"
+    resource: str

main.py

@@ -0,0 +1,3 @@
+"""Entrypoint for CIDC cloud functions."""
+
+from functions import ingest_upload

requirements.dev.txt

@@ -1,2 +1,3 @@
 black==19.3b0
+python-dotenv==0.10.3
 pytest==5.0.1

tests/functions/test_uploads.py

@@ -0,0 +1,9 @@
+from tests.util import make_pubsub_event
+from functions.uploads import ingest_upload
+
+
+def test_ingest_upload():
+    """Test stub event-processing functionality"""
+    job_id = "1"
+    successful_upload_event = make_pubsub_event(job_id)
+    ingest_upload(successful_upload_event, None)

tests/functions/test_util.py

@@ -0,0 +1,9 @@
+from tests.util import make_pubsub_event
+from functions.util import extract_pubsub_data
+
+
+def test_extract_pubsub_data():
+    """Ensure that extract_pubsub_data can do what it claims"""
+    data = "hello there"
+    event = make_pubsub_event(data)
+    assert extract_pubsub_data(event) == data

tests/test_main.py

@@ -1,2 +1,6 @@
+import pytest
+
+
+@pytest.mark.skip("nothing to test for the top-level module, currently")
 def test_main():
     pass

tests/util.py

@@ -0,0 +1,8 @@
+import base64
+
+
+def make_pubsub_event(data: str) -> dict:
+    """Make pubsub event dictionary with base64-encoded data."""
+    b64data = base64.encodebytes(bytes(data, "utf-8"))
+    return {"data": b64data}
+