[INFRA] test docker builds in CI

.github/workflows/bootstrap.yml

@@ -0,0 +1,72 @@
+# this workflow bootstraps the testing of the build the docker images
+#
+# - this will run the python script used to generate the workflows
+#   based on a the jinja template
+# - commit and push the generated workflows to the branch test_docker_build
+#   where they will be executed
+
+name: bootstrap
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [ "master" ]
+
+# Uses the cron schedule for github actions
+#
+# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#scheduled-events
+#
+#         ┌───────────── minute (0 - 59)
+#         │ ┌───────────── hour (0 - 23)
+#         │ │   ┌───────────── day of the month (1 - 31)
+#         │ │   │    ┌───────────── month (1 - 12 or JAN-DEC)
+#         │ │   │    │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
+#         │ │   │    │ │
+#         │ │   │    │ │
+#         │ │   │    │ │
+#         * *   *    * *
+
+  schedule:
+  - cron: 0 0 1,15 * *
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+permissions:
+  contents: write
+jobs:
+  bootstrap:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: python -m pip install jinja2 pyyaml
+
+      - name: Create workflows
+        run: |
+          python .github/workflows/create_workflows.py
+          ls -l .github/workflows
+
+      - name: Push workflows to branch test_docker_build
+        # action: https://github.com/marketplace/actions/git-auto-commit
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: add workflows
+          branch: test_docker_build
+          commit_options: '--no-verify --signoff'
+          commit_user_name: github-actions
+          commit_user_email: [email protected]
+          push_options: '--force'
+          skip_dirty_check: true
+          skip_fetch: true
+          skip_checkout: true
+          create_branch: true

.github/workflows/create_workflows.py

@@ -0,0 +1,195 @@
+"""
+This scripts uses a jinja template to create CI workflows to test.
+
+    - different linux distributions (split by the package manager they use)
+    - different softwares that neurodocker supports
+    - different install method for a given software
+
+All of those are defined in a python dictionary.
+
+Versions to install are read from the neurodocker template for a given software.
+It is possible to skip a version by adding a "skip_versions" key to the software.
+
+Each workflow:
+
+    - installs the latest version of neurodocker
+    - builds a dockerfile for a combination of OS / software / version / install method
+    - cat the dockerfile
+    - attempts to build the corresponding image
+
+This script will also create a "dashboard" saved in docs/README.md
+to be picked up to be rendered by the github pages.
+This requires for you to build the pages from the docs folder
+and on the branch where the workflows are pushed to (currently "test_docker_build").
+
+"""
+from pathlib import Path
+
+import yaml  # type: ignore
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+apt_based = [
+    "ubuntu:22.04",
+    "ubuntu:18.04",
+    "ubuntu:16.04",
+    "debian:bullseye-slim",
+    "debian:buster-slim",
+    "debian:stretch-slim",
+]
+yum_based = ["fedora:36", "centos:7"]
+
+"""
+Add a "skip_versions" key to the software dictionary if you want to skip
+testing a specific version. For example, if you want to skip testing
+version 1.0.0 of afni, add the following to the software dictionary:
+
+    "afni": {
+            "skip_versions": ["1.0.0"],
+            "methods": ["binaries", "source"],
+            "afni_python": ["true", "false"],
+        },
+
+"""
+softwares: dict[str, dict[str, list[str]]] = {
+    "afni": {
+        "methods": ["binaries", "source"],
+        "afni_python": ["true", "false"],
+    },
+    "ants": {
+        "methods": ["binaries", "source"],
+    },
+    "cat12": {"methods": ["binaries"]},
+    "convert3d": {"methods": ["binaries"]},
+    "dcm2niix": {
+        "methods": ["binaries", "source"],
+    },
+    "freesurfer": {"methods": []},
+    "fsl": {
+        "methods": ["binaries"],
+    },
+    "matlabmcr": {
+        "methods": ["binaries"],
+    },
+    "mricron": {"methods": ["binaries"]},
+    "mrtrix3": {
+        "methods": ["binaries", "source"],
+    },
+    "spm12": {"methods": ["binaries"]},
+}
+
+output_dir = Path(__file__).parent
+
+template_folder = Path(__file__).parents[2].joinpath("neurodocker", "templates")
+
+build_dashboard_file = Path(__file__).parents[2].joinpath("docs", "README.md")
+
+# this has to match the name of the branch where the workflows are pushed to
+# see .github/workflows/bootstrap.yml
+branch = "test_docker_build"
+
+# Update to match your username and repo name if you are testing things on your fork
+# "ReproNim/neurodocker"
+repo = "Remi-Gau/neurodocker"
+
+
+def create_dashboard_file():
+    """Create a build dashboard file."""
+
+    print("creating build dashboard file...")
+    print(build_dashboard_file)
+
+    gh_actions_url = "http://github-actions.40ants.com/"
+
+    with open(build_dashboard_file, "w") as f:
+        image_base_url = f"{gh_actions_url}{repo}/matrix.svg?branch={branch}"
+        print(
+            """<!-- This page is generated automatically. Do not edit manually. -->
+# Build dashboard
+""",
+            file=f,
+        )
+
+        # table of content
+        for software, _ in softwares.items():
+            print(f"""- [{software}](#{software})""", file=f)
+
+        print("", file=f)
+
+        # link to the github actions workflow and image of the build status
+        for software, _ in softwares.items():
+            image_url = f"{image_base_url}&only={software}"
+            print(
+                f"""## {software}
+
+[{software} workflow](https://github.com/{repo}/actions/workflows/{software}.yml)
+
+![{software} build status]({image_url})
+""",
+                file=f,
+            )
+
+
+def get_versions_from_neurodocker_template(software: str) -> list[str]:
+    """Load the list of versions to test from the software template."""
+    template = template_folder.joinpath(software).with_suffix(".yaml")
+    with open(template, "r") as f:
+        data = yaml.load(f, Loader=yaml.FullLoader)
+    return list(data["binaries"]["urls"].keys())
+
+
+def stringify(some_list: list[str]) -> str:
+    if len(some_list) == 1:
+        return f"'{some_list[0]}'"
+    return "'" + "', '".join(some_list) + "'"
+
+
+def main():
+    env = Environment(
+        loader=FileSystemLoader(Path(__file__).parent),
+        autoescape=select_autoescape(),
+        lstrip_blocks=True,
+        trim_blocks=True,
+    )
+
+    template = env.get_template("docker_build.jinja")
+
+    os = {
+        "apt_based": stringify(apt_based),
+        "yum_based": stringify(yum_based),
+        "all": stringify(apt_based + yum_based),
+    }
+
+    for software, spec in softwares.items():
+        wf = {
+            "header": "# This is file is automatically generated. Do not edit.",
+            "os": os,
+            "software": software,
+        }
+
+        versions = get_versions_from_neurodocker_template(software)
+        for i in spec.get("skip_versions", []):
+            versions.remove(i)
+
+        if versions is not None and len(versions) > 0:
+            wf["add_version"] = True
+            wf["versions"] = stringify(versions)
+
+        if spec.get("methods") is not None and len(spec["methods"]) > 0:
+            wf["add_method"] = True
+            wf["methods"] = stringify(spec["methods"])
+
+        if spec.get("afni_python") is not None and len(spec["afni_python"]) > 0:
+            wf["add_afni_python"] = True
+            wf["afni_python"] = stringify(spec["afni_python"])
+
+        output_file = output_dir.joinpath(software).with_suffix(".yml")
+        print("creating workflow")
+        print(f"{output_file}")
+        with open(output_file, "w") as f:
+            print(template.render(wf=wf), file=f)
+
+    create_dashboard_file()
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/docker_build.jinja

@@ -0,0 +1,78 @@
+{{ wf.header }}
+name: '{{ wf.software }}'
+
+concurrency:
+  group: ${{ '{{' }} github.workflow {{ '}}' }}-${{ '{{' }} github.ref {{ '}}' }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: ["test_docker_build"]
+
+jobs:
+
+  {{ wf.software }}:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        base_image: [{{ wf.os.all }}]
+        {% if wf.add_version %}
+        version: [{{ wf.versions }}]
+        {% endif %}
+        {% if wf.add_method %}
+        method: [{{ wf.methods }}]
+        {% endif %}
+        {% if wf.add_afni_python %}
+        afni_python: [{{ wf.afni_python }}]
+        {% endif %}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+
+      - name: Install neurodocker
+        run: python -m pip install --editable .[dev]
+
+      - name: Generate Dockerfile
+        run: |
+
+          apt_based=({{ wf.os.apt_based }})
+          if [[ " ${apt_based[*]} " =~ ${{ '{{' }} matrix.base_image {{ '}}' }} ]]; then
+            pkg_manager="apt"
+          fi
+
+          yum_based=({{ wf.os.yum_based }})
+          if [[ " ${yum_based[*]} " =~ ${{ '{{' }} matrix.base_image {{ '}}' }} ]]; then
+            pkg_manager="yum"
+          fi
+
+          neurodocker \
+            generate docker \
+            --base-image=${{ '{{' }} matrix.base_image {{ '}}' }} \
+            --pkg-manager=${pkg_manager} \
+            --yes \
+{% if wf.software == 'cat12' %}
+            --matlabmcr method='binaries' version='2017b' \
+{% endif %}
+            --{{ wf.software }} \
+{% if wf.add_version %}
+            version=${{ '{{' }} matrix.version {{ '}}' }} \
+{% endif %}
+{% if wf.add_method %}
+            method=${{ '{{' }} matrix.method {{ '}}' }} \
+{% endif %}
+{% if wf.add_afni_python %}
+            install_python3=${{ '{{' }} matrix.afni_python {{ '}}' }} \
+{% endif %}            > Dockerfile_tmp
+
+          cat Dockerfile_tmp
+
+      - name: Build the Docker image
+        run: docker build -f Dockerfile_tmp .

.github/workflows/docs.yml

@@ -3,7 +3,7 @@ on:
     branches:
     - master
 permissions:
-  contents: write  
+  contents: write
 jobs:
   build_docs:
     runs-on: ubuntu-latest

.gitignore

@@ -139,4 +139,4 @@ dmypy.json
 cython_debug/
 
 # Pycharm
-.idea/
+.idea/

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: check-yaml
     -   id: check-added-large-files
 -   repo: https://github.com/psf/black
-    rev: "22.12.0"
+    rev: "23.1.0"
     hooks:
     -   id: black
 

README.md

@@ -15,11 +15,12 @@ Please see our website https://www.repronim.org/neurodocker for more information
 Use the _Neurodocker_ Docker image (recommended):
 
 ```shell
-docker run --rm repronim/neurodocker:0.7.0 --help
+docker run --rm repronim/neurodocker:latest --help
 ```
 
 The Docker images were moved to [repronim/neurodocker](https://hub.docker.com/r/repronim/neurodocker) from [kaczmarj/neurodocker](https://hub.docker.com/r/kaczmarj/neurodocker).
 
+
 This project can also be installed with `pip`:
 
 ```shell
@@ -47,3 +48,10 @@ python -m pip install --no-cache-dir --editable .[all]
 ```
 
 Before committing changes, initialize `pre-commit` with `pre-commit install`. This will format code with each commit to keep the style consistent. _Neurodocker_ uses `black` for formatting.
+
+
+## Build status
+
+You can check the status of the build of the Docker images
+for several of the neuroimaging software packages that are supported by _Neurodocker_
+on [this page](build_dashboard.md).