Add CICD schema and render workflows

.github/workflows/test-provider.yaml

@@ -36,6 +36,10 @@ jobs:
           - do
           - gcp
           - local
+        cicd:
+          - none
+          - github-actions
+          - gitlab-ci
     steps:
       - name: 'Checkout Infrastructure'
         uses: actions/checkout@main
@@ -65,13 +69,14 @@ jobs:
           pip install .[dev]
       - name: QHub Initialize
         run: |
-          qhub init "${{ matrix.provider }}" --project "test-${{ matrix.provider }}" --domain "${{ matrix.provider }}.qhub.dev" --auth-provider github --disable-prompt
+          qhub init "${{ matrix.provider }}" --project "test--${{ matrix.provider }}-${{ matrix.cicd }}" --domain "${{ matrix.provider }}.qhub.dev" --auth-provider github --disable-prompt --ci-provider ${{ matrix.cicd }}
+          cat "qhub-config.yaml"
       - name: QHub Render
         run: |
-          qhub render -c "qhub-config.yaml" -o "qhub-${{ matrix.provider }}-deployment"
-          cp "qhub-config.yaml" "qhub-${{ matrix.provider }}-deployment/qhub-config.yaml"
+          qhub render -c "qhub-config.yaml" -o "qhub-${{ matrix.provider }}-${{ matrix.cicd }}-deployment"
+          cp "qhub-config.yaml" "qhub-${{ matrix.provider }}-${{ matrix.cicd }}-deployment/qhub-config.yaml"
       - name: QHub Render Artifact
         uses: actions/upload-artifact@master
         with:
-          name: "qhub-${{ matrix.provider }}-artifact"
-          path: "qhub-${{ matrix.provider }}-deployment"
+          name: "qhub-${{ matrix.provider }}-${{ matrix.cicd }}-artifact"
+          path: "qhub-${{ matrix.provider }}-${{ matrix.cicd }}-deployment"

qhub/provider/cicd/github.py

@@ -1,9 +1,15 @@
 import os
 import base64
 
+from typing import Optional, Dict, List, Union
+
+from pydantic import BaseModel, Field
+
 import requests
 from nacl import encoding, public
 
+from qhub.utils import pip_install_qhub
+
 
 def github_request(url, method="GET", json=None):
     GITHUB_BASE_URL = "https://api.github.com/"
@@ -82,3 +88,219 @@ def create_repository(owner, repo, description, homepage, private=True):
             },
         )
     return f"[email protected]:{owner}/{repo}.git"
+
+
+def gha_env_vars(config):
+    env_vars = {
+        "GITHUB_TOKEN": "${{ secrets.GITHUB_TOKEN }}",
+    }
+
+    if config["provider"] == "aws":
+        env_vars["AWS_ACCESS_KEY_ID"] = "${{ secrets.AWS_ACCESS_KEY_ID }}"
+        env_vars["AWS_SECRET_ACCESS_KEY"] = "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+        env_vars["AWS_DEFAULT_REGION"] = "${{ secrets.AWS_DEFAULT_REGION }}"
+    elif config["provider"] == "azure":
+        env_vars["ARM_CLIENT_ID"] = "${{ secrets.ARM_CLIENT_ID }}"
+        env_vars["ARM_CLIENT_SECRET"] = "${{ secrets.ARM_CLIENT_SECRET }}"
+        env_vars["ARM_SUBSCRIPTION_ID"] = "${{ secrets.ARM_SUBSCRIPTION_ID }}"
+        env_vars["ARM_TENANT_ID"] = "${{ secrets.ARM_TENANT_ID }}"
+    elif config["provider"] == "do":
+        env_vars["AWS_ACCESS_KEY_ID"] = "${{ secrets.AWS_ACCESS_KEY_ID }}"
+        env_vars["AWS_SECRET_ACCESS_KEY"] = "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+        env_vars["SPACES_ACCESS_KEY_ID"] = "${{ secrets.SPACES_ACCESS_KEY_ID }}"
+        env_vars["SPACES_SECRET_ACCESS_KEY"] = "${{ secrets.SPACES_SECRET_ACCESS_KEY }}"
+        env_vars["DIGITALOCEAN_TOKEN"] = "${{ secrets.DIGITALOCEAN_TOKEN }}"
+    elif config["provider"] == "gcp":
+        env_vars["GOOGLE_CREDENTIALS"] = "${{ secrets.GOOGLE_CREDENTIALS }}"
+    elif config["provider"] == "local":
+        # create mechanism to allow for extra env vars?
+        pass
+    else:
+        raise ValueError("Cloud Provider configuration not supported")
+
+    return env_vars
+
+
+### GITHUB-ACTIONS SCHEMA ###
+
+
+class GHA_on_extras(BaseModel):
+    branches: List[str]
+    path: List[str]
+
+
+class GHA_on(BaseModel):
+    # to allow for dynamic key names
+    __root__: Dict[str, GHA_on_extras]
+
+    # TODO: validate __root__ values
+    # `push`, `pull_request`, etc.
+
+
+class GHA_job_steps_extras(BaseModel):
+    # to allow for dynamic key names
+    __root__: Union[str, float, int]
+
+
+class GHA_job_step(BaseModel):
+    name: str
+    uses: Optional[str]
+    with_: Optional[Dict[str, GHA_job_steps_extras]] = Field(alias="with")
+    run: Optional[str]
+    env: Optional[Dict[str, GHA_job_steps_extras]]
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class GHA_job_id(BaseModel):
+    name: str
+    runs_on_: str = Field(alias="runs-on")
+    steps: List[GHA_job_step]
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class GHA_jobs(BaseModel):
+    # to allow for dynamic key names
+    __root__: Dict[str, GHA_job_id]
+
+
+class GHA(BaseModel):
+    name: str
+    on: GHA_on
+    env: Optional[Dict[str, str]]
+    jobs: List[GHA_jobs]
+
+
+class QhubOps(GHA):
+    pass
+
+
+class QhubLinter(GHA):
+    pass
+
+
+### GITHUB ACTION WORKFLOWS ###
+
+PYTHON_VERSION = 3.8
+
+
+def checkout_image_step():
+    return GHA_job_step(
+        name="Checkout Image",
+        uses="actions/checkout@master",
+        with_={
+            "token": GHA_job_steps_extras(
+                __root__="{{ '${{ secrets.REPOSITORY_ACCESS_TOKEN }}' }}"
+            )
+        },
+    )
+
+
+def setup_python_step():
+    return GHA_job_step(
+        name="Set up Python",
+        uses="actions/setup-python@v2",
+        with_={"python-version": GHA_job_steps_extras(__root__=PYTHON_VERSION)},
+    )
+
+
+def install_qhub_step(qhub_version):
+    return GHA_job_step(name="Install QHub", run=pip_install_qhub(qhub_version))
+
+
+def gen_qhub_ops(config):
+
+    env_vars = gha_env_vars(config)
+    branch = config["ci_cd"]["branch"]
+    qhub_version = config["qhub_version"]
+
+    push = GHA_on_extras(branches=[branch], path=["qhub-config.yaml"])
+    on = GHA_on(__root__={"push": push})
+
+    step1 = checkout_image_step()
+    step2 = setup_python_step()
+    step3 = install_qhub_step(qhub_version)
+
+    step4 = GHA_job_step(
+        name="Deploy Changes made in qhub-config.yaml",
+        run=f"qhub deploy -c qhub-config.yaml --disable-prompt{' --skip-remote-state-provision' if os.environ.get('QHUB_GH_BRANCH') else ''}",
+    )
+
+    step5 = GHA_job_step(
+        name="Push Changes",
+        run=(
+            "git config user.email '[email protected]' ; "
+            "git config user.name 'github action' ; "
+            "git add . ; "
+            "git diff --quiet && git diff --staged --quiet || (git commit -m '${COMMIT_MSG}') ; "
+            f"git push origin {branch}"
+        ),
+        env={
+            "COMMIT_MSG": GHA_job_steps_extras(
+                __root__="qhub-config.yaml automated commit: {{ '${{ github.sha }}' }}"
+            )
+        },
+    )
+
+    job1 = GHA_job_id(
+        name="qhub", runs_on_="ubuntu-latest", steps=[step1, step2, step3, step4, step5]
+    )
+    jobs = [GHA_jobs(__root__={"build": job1})]
+
+    return QhubOps(
+        name="qhub auto update",
+        on=on,
+        env=env_vars,
+        jobs=jobs,
+    )
+
+
+def gen_qhub_linter(config):
+
+    env_vars = {}
+    qhub_gh_branch = os.environ.get("QHUB_GH_BRANCH")
+    if qhub_gh_branch:
+        env_vars["QHUB_GH_BRANCH"] = qhub_gh_branch
+    else:
+        env_vars = None
+
+    branch = config["ci_cd"]["branch"]
+    qhub_version = config["qhub_version"]
+
+    pull_request = GHA_on_extras(branches=[branch], path=["qhub-config.yaml"])
+    on = GHA_on(__root__={"pull_request": pull_request})
+
+    step1 = checkout_image_step()
+    step2 = setup_python_step()
+    step3 = install_qhub_step(qhub_version)
+
+    step4_envs = {
+        "PR_NUMBER": GHA_job_steps_extras(
+            __root__="{{ '${{ github.event.number }}' }}"
+        ),
+        "REPO_NAME": GHA_job_steps_extras(__root__="{{ '${{ github.repository }}' }}"),
+        "GITHUB_TOKEN": GHA_job_steps_extras(
+            __root__="{{ '${{ secrets.REPOSITORY_ACCESS_TOKEN }}' }}"
+        ),
+    }
+
+    step4 = GHA_job_step(
+        name="QHub Lintify",
+        run="qhub validate --config qhub-config.yaml --enable-commenting",
+        env=step4_envs,
+    )
+
+    job1 = GHA_job_id(
+        name="qhub", runs_on_="ubuntu-latest", steps=[step1, step2, step3, step4]
+    )
+    jobs = [GHA_jobs(__root__={"qhub-validate": job1})]
+
+    return QhubLinter(
+        name="qhub linter",
+        on=on,
+        env=env_vars,
+        jobs=jobs,
+    )

qhub/provider/cicd/gitlab.py

@@ -0,0 +1,86 @@
+from typing import Optional, Dict, List, Union
+
+from pydantic import BaseModel, Field
+from qhub.utils import pip_install_qhub
+
+
+class GLCI_extras(BaseModel):
+    # to allow for dynamic key names
+    __root__: Union[str, float, int]
+
+
+class GLCI_image(BaseModel):
+    name: str
+    entrypoint: Optional[str]
+
+
+class GLCI_rules(BaseModel):
+    if_: Optional[str] = Field(alias="if")
+    changes: Optional[List[str]]
+    # exists:
+    # variables:
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class GLCI_job(BaseModel):
+    image: Optional[Union[str, GLCI_image]]
+    variables: Optional[Dict[str, str]]
+    before_script: Optional[List[str]]
+    after_script: Optional[List[str]]
+    script: List[str]
+    rules: GLCI_rules
+
+
+class GLCI(BaseModel):
+    __root__: Dict[str, GLCI_job]
+
+
+PYTHON_VERSION = 3.9
+
+
+def gen_gitlab_ci(config):
+
+    branch = config["ci_cd"]["branch"]
+    before_script = config["ci_cd"].get("before_script")
+    after_script = config["ci_cd"].get("after_script")
+    pip_install = pip_install_qhub(config["qhub_version"])
+
+    render_vars = {
+        "COMMIT_MSG": "qhub-config.yaml automated commit: {{ '$CI_COMMIT_SHA' }}",
+    }
+
+    # if qhub_gh_branch:
+    #     render_vars["QHUB_GH_BRANCH"] = qhub_gh_branch
+    #     pip_install_qhub = f"pip install https://github.com/Quansight/qhub/archive/{qhub_gh_branch}.zip"
+
+    script = [
+        f"git checkout {branch}",
+        f"{pip_install}",
+        "qhub deploy --config qhub-config.yaml --disable-prompt --skip-remote-state-provision",
+        "git config user.email '[email protected]'",
+        "git config user.name 'gitlab ci'",
+        "git add .",
+        "git diff --quiet && git diff --staged --quiet || (git commit -m '${COMMIT_MSG}'; git push origin {branch})",
+    ]
+
+    rules = GLCI_rules(
+        if_=f"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == '{branch}'",
+        changes=["qhub-config.yaml"],
+    )
+
+    render_qhub = GLCI_job(
+        image=f"python:{PYTHON_VERSION}",
+        variables=render_vars,
+        before_script=before_script,
+        after_script=after_script,
+        script=script,
+        rules=rules,
+    )
+
+    return GLCI(
+        __root__={
+            "render-qhub": render_qhub,
+        }
+    )