Update build CI (#713)

holoviz · Mar 20, 2024 · 2c83c20 · 2c83c20
1 parent 5a7dceb
commit 2c83c20
Show file tree

Hide file tree

Showing 6 changed files with 206 additions and 127 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -2,158 +2,220 @@ name: packages
 on:
   push:
     tags:
-    - 'v[0-9]+.[0-9]+.[0-9]+'
-    - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+'
-    - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+'
-    - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
+      - "v[0-9]+.[0-9]+.[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+a[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+b[0-9]+"
+      - "v[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
   # Dry-run only
   workflow_dispatch:
   schedule:
-    - cron: '0 18 * * SUN'
+    - cron: "0 18 * * SUN"
+
+defaults:
+  run:
+    shell: bash -el {0}
 
 env:
-  PYTHON_VERSION: "3.9"
-  NODE_VERSION: "18"
+  PACKAGE: "geoviews"
+  PYTHON_VERSION: "3.11"
+  NODE_VERSION: "20"
   MPLBACKEND: "Agg"
   SETUPTOOLS_ENABLE_FEATURES: "legacy-editable"
 
 jobs:
+  waiting_room:
+    name: Waiting Room
+    runs-on: ubuntu-latest
+    needs: [conda_build, pip_install, npm_build]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    environment:
+      name: publish
+    steps:
+      - run: echo "All builds have finished, have been approved, and ready to publish"
+
   conda_build:
-    name: Build Conda Packages
-    runs-on: 'ubuntu-latest'
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      CHANS_DEV: "-c pyviz/label/dev -c bokeh -c conda-forge -c nodefaults"
-      PKG_TEST_PYTHON: "--test-python=py39"
-      CONDA_UPLOAD_TOKEN: ${{ secrets.CONDA_UPLOAD_TOKEN }}
+    name: Build Conda
+    runs-on: "ubuntu-latest"
     steps:
-      - name: remove nodejs
-        run: |
-          sudo rm /usr/local/bin/node
-          sudo rm /usr/local/bin/npm
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "100"
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - uses: conda-incubator/setup-miniconda@v2
+      - name: Fetch unshallow
+        run: git fetch --prune --tags --unshallow -f
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           miniconda-version: "latest"
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
-      - name: Fetch unshallow
-        run: git fetch --prune --tags --unshallow -f
-      - name: Set output
-        id: vars
-        run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
       - name: conda setup
         run: |
-          conda config --set always_yes True
-          conda install -n base conda-libmamba-solver
-          conda install -n test conda-libmamba-solver
-          conda config --set solver libmamba
-          conda install -c pyviz "pyctdev>=0.5"
-          doit ecosystem_setup
-          conda install -c conda-forge "urllib3<2.0.0" "conda-build==3.24"
+          conda install -y conda-build build pyct
       - name: conda build
         run: |
-          doit package_build --recipe=core $CHANS_DEV $PKG_TEST_PYTHON --test-group=simple
-          doit package_build --recipe=recommended $CHANS_DEV $PKG_TEST_PYTHON --test-group=simple
-      - name: npm setup
-        run: |
-          echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_API_TOKEN }}" > $HOME/.npmrc
-          npm whoami
-          npm -v
-      - name: dev deploy
-        if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: |
-          doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev --recipe=core
-          doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev --recipe=recommended
-      - name: main deploy
-        if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: |
-          doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev --label=main --recipe=core
-          doit package_upload --token=$CONDA_UPLOAD_TOKEN --label=dev --label=main --recipe=recommended
-      - name: create env
-        run: |
-          doit env_create $CHANS_DEV --python=$PYTHON_VERSION
-      - name: build js
+          source ./scripts/conda/build.sh
+          echo "CONDA_FILE="$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2"" >> $GITHUB_ENV
+          echo "CONDA_CORE_FILE="$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-core-$VERSION-py_0.tar.bz2"" >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: conda
+          path: |
+            ${{ env.CONDA_FILE }}
+            ${{ env.CONDA_CORE_FILE }}
+          if-no-files-found: error
+
+  conda_publish:
+    name: Publish Conda
+    runs-on: ubuntu-latest
+    needs: [conda_build, waiting_room]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: conda
+          path: dist/
+      - name: Set environment variables
+        run: |
+          echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "CONDA_FILE=$(ls dist/*.tar.bz2 | grep -v core)" >> $GITHUB_ENV
+          echo "CONDA_CORE_FILE=$(ls dist/*-core-*.tar.bz2)" >> $GITHUB_ENV
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+      - name: conda setup
         run: |
-          eval "$(conda shell.bash hook)"
-          conda install -n test-environment conda-libmamba-solver
-          conda activate test-environment
-          doit develop_install $CHANS_DEV -o build
-          python setup.py develop
-      - name: npm dev deploy
-        if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
+          conda install -y anaconda-client
+      - name: conda dev upload
+        if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')
         run: |
-          cd ./geoviews
-          npm publish --tag dev --access public
-      - name: npm main deploy
-        if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_CORE_FILE
+      - name: conda main upload
+        if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')))
         run: |
-          cd ./geoviews
-          npm publish --tag latest
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE
+          anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_CORE_FILE
+
   pip_build:
-    name: Build PyPI Packages
-    runs-on: 'ubuntu-latest'
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      CHANS_DEV: "-c pyviz/label/dev -c bokeh -c conda-forge"
-      PKG_TEST_PYTHON: "--test-python=py39"
-      CHANS: "-c pyviz"
-      PPU: ${{ secrets.PPU }}
-      PPP: ${{ secrets.PPP }}
-      PYPI: "https://upload.pypi.org/legacy/"
+    name: Build PyPI
+    runs-on: "ubuntu-latest"
     steps:
-      - name: remove nodejs
-        run: |
-          sudo rm /usr/local/bin/node
-          sudo rm /usr/local/bin/npm
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: "100"
-      - uses: actions/setup-python@v4
+      - name: Fetch unshallow
+        run: git fetch --prune --tags --unshallow -f
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: actions/setup-node@v4
         with:
-          miniconda-version: "latest"
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Install build
+        run: |
+          python -m pip install build
+      - name: Build package
+        run: python -m build .
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pip
+          path: dist/
+          if-no-files-found: error
+
+  pip_install:
+    name: Install PyPI
+    runs-on: "ubuntu-latest"
+    needs: [pip_build]
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: pip
+          path: dist/
+      - name: Install package
+        run: python -m pip install dist/*.whl
+      - name: Test package
+        run: python -c "import $PACKAGE; print($PACKAGE.__version__)"
+
+  pip_publish:
+    name: Publish PyPI
+    runs-on: ubuntu-latest
+    needs: [pip_build, waiting_room]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: pip
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: ${{ secrets.PPU }}
+          password: ${{ secrets.PPP }}
+          repository-url: "https://upload.pypi.org/legacy/"
+
+  npm_build:
+    name: Build NPM
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "100"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
       - name: Fetch unshallow
         run: git fetch --prune --tags --unshallow -f
-      - name: conda setup
-        run: |
-          conda config --set always_yes True
-          conda install -c pyviz "pyctdev>=0.5"
-          doit ecosystem_setup
-          doit env_create $CHANS_DEV --python=$PYTHON_VERSION
-      - name: env setup
+      - name: package install
+        run: |
+          python -m pip install -ve .
+      - name: npm build
+        run: |
+          cd ${{ env.PACKAGE }}
+          TARBALL=$(npm pack .)
+          echo "TARBALL=$TARBALL" >> $GITHUB_ENV
+          npm publish --dry-run $TARBALL
+          cd ..
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: npm
+          if-no-files-found: error
+          path: ${{ env.PACKAGE }}/${{ env.TARBALL }}
+
+  npm_publish:
+    name: Publish NPM
+    runs-on: "ubuntu-latest"
+    needs: [npm_build, waiting_room]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: npm
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Set environment variables
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit develop_install $CHANS_DEV -o build
-          doit pip_on_conda
-      - name: pip build
+          echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "TARBALL=$(ls *.tgz)" >> $GITHUB_ENV
+      - name: npm setup
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=simple
-      - name: git status
+          echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_API_TOKEN }}" > $HOME/.npmrc
+          npm whoami
+          npm -v
+      - name: npm dev deploy
+        if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')
         run: |
-          git status
-          git diff
-      - name: pip upload
-        if: github.event_name == 'push'
+          npm publish --tag dev $TARBALL
+      - name: npm main deploy
+        if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')))
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_upload -u $PPU -p $PPP -r $PYPI
+          npm publish --tag latest $TARBALL
diff --git a/geoviews/package-lock.json b/geoviews/package-lock.json
diff --git a/geoviews/package.json b/geoviews/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@holoviz/geoviews",
-  "version": "1.11.1",
+  "version": "1.12.0-a.2",
   "description": "Simple, concise geographical visualization in Python",
   "license": "BSD-3-Clause",
   "repository": {

diff --git a/scripts/conda/build.sh b/scripts/conda/build.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+git status
+
+export SETUPTOOLS_ENABLE_FEATURES="legacy-editable"
+python -m build -w .
+
+git diff --exit-code
+
+VERSION=$(find dist -name "*.whl" -exec basename {} \; | cut -d- -f2)
+export VERSION
+
+# Note: pyct is needed in the same environment as conda-build!
+conda build scripts/conda/recipe-core --no-anaconda-upload --no-verify
+conda build scripts/conda/recipe-recommended --no-anaconda-upload --no-verify
diff --git a/conda.recipe/core/meta.yaml → scripts/conda/recipe-core/meta.yaml b/conda.recipe/core/meta.yaml → scripts/conda/recipe-core/meta.yaml
@@ -1,15 +1,15 @@
-{% set sdata = load_setup_py_data() %}
+{% set sdata = load_setup_py_data(setup_file="../../../setup.py", from_recipe_dir=True) %}
 
 package:
-  name: geoviews-core
-  version: {{ sdata['version'] }}
+  name: {{ sdata['name'] }}-core
+  version: {{ VERSION }}
 
 source:
-  path: ../..
+  url: ../../../dist/{{ sdata['name'] }}-{{ VERSION }}-py3-none-any.whl
 
 build:
   noarch: python
-  script: python setup.py install --single-version-externally-managed --record=record.txt
+  script: {{ PYTHON }} -m pip install -vv {{ sdata['name'] }}-{{ VERSION }}-py3-none-any.whl
   entry_points:
     {% for group,epoints in sdata.get("entry_points",{}).items() %}
     {% for entry_point in epoints %}