Dev/2.3.0

.github/workflows/tests.yml

@@ -2,6 +2,7 @@
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 # Based on ~/code/xcookie/xcookie/rc/tests.yml.in
 # Now based on ~/code/xcookie/xcookie/builders/github_actions.py
+# See: https://github.com/Erotemic/xcookie
 
 name: PurePyCI
 
@@ -12,12 +13,17 @@ on:
 
 jobs:
   lint_job:
+    ##
+    # Run quick linting and typing checks.
+    # To disable all linting add "linter=false" to the xcookie config.
+    # To disable type checks add "notypes" to the xcookie tags.
+    ##
     runs-on: ubuntu-latest
     steps:
     - name: Checkout source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4.1.1
     - name: Set up Python 3.11 for linting
-      uses: actions/setup-python@v4.5.0
+      uses: actions/setup-python@v5.0.0
       with:
         python-version: '3.11'
     - name: Install dependencies
@@ -29,21 +35,25 @@ jobs:
         # stop the build if there are Python syntax errors or undefined names
         flake8 ./vtool_ibeis --count --select=E9,F63,F7,F82 --show-source --statistics
   build_and_test_sdist:
+    ##
+    # Build the pure python package from source and test it in the
+    # same environment.
+    ##
     name: Build sdist
     runs-on: ubuntu-latest
     steps:
     - name: Checkout source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4.1.1
     - name: Set up Python 3.11
-      uses: actions/setup-python@v4.5.0
+      uses: actions/setup-python@v5.0.0
       with:
         python-version: '3.11'
     - name: Upgrade pip
       run: |-
         python -m pip install --upgrade pip
-        python -m pip install -r requirements/tests.txt
-        python -m pip install -r requirements/runtime.txt
-        python -m pip install -r requirements/headless.txt
+        python -m pip install --prefer-binary -r requirements/tests.txt
+        python -m pip install --prefer-binary -r requirements/runtime.txt
+        python -m pip install --prefer-binary -r requirements/headless.txt
     - name: Build sdist
       shell: bash
       run: |-
@@ -52,7 +62,7 @@ jobs:
         python -m build --sdist --outdir wheelhouse
     - name: Install sdist
       run: |-
-        ls -al ./wheelhouse
+        ls -al wheelhouse
         pip install --prefer-binary wheelhouse/vtool_ibeis*.tar.gz -v
     - name: Test minimal loose sdist
       run: |-
@@ -66,7 +76,7 @@ jobs:
         # Get path to installed package
         MOD_DPATH=$(python -c "import vtool_ibeis, os; print(os.path.dirname(vtool_ibeis.__file__))")
         echo "MOD_DPATH = $MOD_DPATH"
-        python -m pytest --cov={self.mod_name} $MOD_DPATH ../tests
+        python -m pytest --verbose --cov=vtool_ibeis $MOD_DPATH ../tests
         cd ..
     - name: Test full loose sdist
       run: |-
@@ -81,17 +91,22 @@ jobs:
         # Get path to installed package
         MOD_DPATH=$(python -c "import vtool_ibeis, os; print(os.path.dirname(vtool_ibeis.__file__))")
         echo "MOD_DPATH = $MOD_DPATH"
-        python -m pytest --cov={self.mod_name} $MOD_DPATH ../tests
+        python -m pytest --verbose --cov=vtool_ibeis $MOD_DPATH ../tests
         cd ..
-    - name: Upload sdist artifact
-      uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v3.1.3
+      name: Upload sdist artifact
       with:
-        name: wheels
-        path: ./wheelhouse/*.tar.gz
+        name: sdist_wheels
+        path: wheelhouse/*.tar.gz
   build_purepy_wheels:
+    ##
+    # Download and test the pure-python wheels that were build in the
+    # build_purepy_wheels and test them in this independent environment.
+    ##
     name: ${{ matrix.python-version }} on ${{ matrix.os }}, arch=${{ matrix.arch }} with ${{ matrix.install-extras }}
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os:
         - ubuntu-latest
@@ -101,25 +116,26 @@ jobs:
         - auto
     steps:
     - name: Checkout source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4.1.1
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       if: runner.os == 'Linux' && matrix.arch != 'auto'
       with:
         platforms: all
     - name: Setup Python
-      uses: actions/setup-python@v4.5.0
+      uses: actions/setup-python@v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Build pure wheel
       shell: bash
       run: |-
-        python -m pip install setuptools>=0.8 wheel build
+        python -m pip install setuptools>=0.8 wheel build twine
         python -m build --wheel --outdir wheelhouse
+        python -m twine check ./wheelhouse/vtool_ibeis*.whl
     - name: Show built files
       shell: bash
       run: ls -la wheelhouse
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v3.1.3
       name: Upload wheels artifact
       with:
         name: wheels
@@ -130,20 +146,19 @@ jobs:
     needs:
     - build_purepy_wheels
     strategy:
+      fail-fast: false
       matrix:
+        # Xcookie generates an explicit list of environments that will be used
+        # for testing instead of using the more concise matrix notation.
         include:
-        - python-version: '3.7'
+        - python-version: '3.8'
           install-extras: tests-strict,runtime-strict,headless-strict
           os: ubuntu-latest
           arch: auto
         - python-version: '3.11'
           install-extras: tests-strict,runtime-strict,optional-strict,headless-strict
           os: ubuntu-latest
           arch: auto
-        - python-version: '3.7'
-          install-extras: tests,optional,headless
-          os: ubuntu-latest
-          arch: auto
         - python-version: '3.8'
           install-extras: tests,optional,headless
           os: ubuntu-latest
@@ -162,17 +177,17 @@ jobs:
           arch: auto
     steps:
     - name: Checkout source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4.1.1
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       if: runner.os == 'Linux' && matrix.arch != 'auto'
       with:
         platforms: all
     - name: Setup Python
-      uses: actions/setup-python@v4.5.0
+      uses: actions/setup-python@v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v2.1.1
       name: Download wheels
       with:
         name: wheels
@@ -189,6 +204,9 @@ jobs:
         pip install tomli pkginfo
         export WHEEL_FPATH=$(python -c "import pathlib; print(str(sorted(pathlib.Path('wheelhouse').glob('vtool_ibeis*.whl'))[-1]).replace(chr(92), chr(47)))")
         export MOD_VERSION=$(python -c "from pkginfo import Wheel; print(Wheel('$WHEEL_FPATH').version)")
+        echo "$WHEEL_FPATH=WHEEL_FPATH"
+        echo "$INSTALL_EXTRAS=INSTALL_EXTRAS"
+        echo "$MOD_VERSION=MOD_VERSION"
         pip install --prefer-binary "vtool_ibeis[$INSTALL_EXTRAS]==$MOD_VERSION" -f wheelhouse
         echo "Install finished."
     - name: Test wheel ${{ matrix.install-extras }}
@@ -206,14 +224,15 @@ jobs:
         ls -altr
         # Get the path to the installed package and run the tests
         export MOD_DPATH=$(python -c "import vtool_ibeis, os; print(os.path.dirname(vtool_ibeis.__file__))")
+        export MOD_NAME=vtool_ibeis
         echo "
         ---
         MOD_DPATH = $MOD_DPATH
         ---
         running the pytest command inside the workspace
         ---
         "
-        python -m pytest -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --cov="vtool_ibeis" "$MOD_DPATH" ../tests
+        python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --durations=100 --cov="$MOD_NAME" "$MOD_DPATH" ../tests
         echo "pytest command finished, moving the coverage file to the repo root"
         ls -al
         # Move coverage file to a new name
@@ -236,10 +255,11 @@ jobs:
         echo '### The cwd should now have a coverage.xml'
         ls -altr
         pwd
-    - uses: codecov/codecov-action@v3
+    - uses: codecov/codecov-action@v4.0.1
       name: Codecov Upload
       with:
         file: ./coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}
   test_deploy:
     name: Uploading Test to PyPi
     runs-on: ubuntu-latest
@@ -250,19 +270,24 @@ jobs:
     - test_purepy_wheels
     steps:
     - name: Checkout source
-      uses: actions/checkout@v3
-    - uses: actions/download-artifact@v3
-      name: Download wheels and sdist
+      uses: actions/checkout@v4.1.1
+    - uses: actions/download-artifact@v2.1.1
+      name: Download wheels
       with:
         name: wheels
         path: wheelhouse
+    - uses: actions/[email protected]
+      name: Download sdist
+      with:
+        name: sdist_wheels
+        path: wheelhouse
     - name: Show files to upload
       shell: bash
       run: ls -la wheelhouse
     - name: Sign and Publish
       env:
         TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
-        TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }}
+        TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }}
         CI_SECRET: ${{ secrets.CI_SECRET }}
       run: |-
@@ -283,7 +308,33 @@ jobs:
         pip install urllib3 requests[security] twine
         GPG_KEYID=$(cat dev/public_gpg_key)
         echo "GPG_KEYID = '$GPG_KEYID'"
-        DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE DO_UPLOAD=True DO_TAG=False ./publish.sh
+        GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID"
+        WHEEL_PATHS=(wheelhouse/*.whl wheelhouse/*.tar.gz)
+        WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}")
+        echo "$WHEEL_PATHS_STR"
+        for WHEEL_PATH in "${WHEEL_PATHS[@]}"
+        do
+            echo "------"
+            echo "WHEEL_PATH = $WHEEL_PATH"
+            $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH
+            $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH  || echo "hack, the first run of gpg very fails"
+            $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH
+        done
+        ls -la wheelhouse
+        pip install opentimestamps-client
+        ots stamp wheelhouse/*.whl wheelhouse/*.tar.gz wheelhouse/*.asc
+        ls -la wheelhouse
+        twine upload --username __token__ --password "$TWINE_PASSWORD" --repository-url "$TWINE_REPOSITORY_URL" wheelhouse/*.whl wheelhouse/*.tar.gz --skip-existing --verbose || { echo "failed to twine upload" ; exit 1; }
+    - uses: actions/[email protected]
+      name: Upload deploy artifacts
+      with:
+        name: deploy_artifacts
+        path: |-
+          wheelhouse/*.whl
+          wheelhouse/*.zip
+          wheelhouse/*.tar.gz
+          wheelhouse/*.asc
+          wheelhouse/*.ots
   live_deploy:
     name: Uploading Live to PyPi
     runs-on: ubuntu-latest
@@ -294,19 +345,24 @@ jobs:
     - test_purepy_wheels
     steps:
     - name: Checkout source
-      uses: actions/checkout@v3
-    - uses: actions/download-artifact@v3
-      name: Download wheels and sdist
+      uses: actions/checkout@v4.1.1
+    - uses: actions/download-artifact@v2.1.1
+      name: Download wheels
       with:
         name: wheels
         path: wheelhouse
+    - uses: actions/[email protected]
+      name: Download sdist
+      with:
+        name: sdist_wheels
+        path: wheelhouse
     - name: Show files to upload
       shell: bash
       run: ls -la wheelhouse
     - name: Sign and Publish
       env:
         TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/
-        TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+        TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
         CI_SECRET: ${{ secrets.CI_SECRET }}
       run: |-
@@ -327,7 +383,77 @@ jobs:
         pip install urllib3 requests[security] twine
         GPG_KEYID=$(cat dev/public_gpg_key)
         echo "GPG_KEYID = '$GPG_KEYID'"
-        DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE DO_UPLOAD=True DO_TAG=False ./publish.sh
+        GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID"
+        WHEEL_PATHS=(wheelhouse/*.whl wheelhouse/*.tar.gz)
+        WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}")
+        echo "$WHEEL_PATHS_STR"
+        for WHEEL_PATH in "${WHEEL_PATHS[@]}"
+        do
+            echo "------"
+            echo "WHEEL_PATH = $WHEEL_PATH"
+            $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH
+            $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH  || echo "hack, the first run of gpg very fails"
+            $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH
+        done
+        ls -la wheelhouse
+        pip install opentimestamps-client
+        ots stamp wheelhouse/*.whl wheelhouse/*.tar.gz wheelhouse/*.asc
+        ls -la wheelhouse
+        twine upload --username __token__ --password "$TWINE_PASSWORD" --repository-url "$TWINE_REPOSITORY_URL" wheelhouse/*.whl wheelhouse/*.tar.gz --skip-existing --verbose || { echo "failed to twine upload" ; exit 1; }
+    - uses: actions/[email protected]
+      name: Upload deploy artifacts
+      with:
+        name: deploy_artifacts
+        path: |-
+          wheelhouse/*.whl
+          wheelhouse/*.zip
+          wheelhouse/*.tar.gz
+          wheelhouse/*.asc
+          wheelhouse/*.ots
+  release:
+    name: Create Github Release
+    if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/tags') || startsWith(github.event.ref, 'refs/heads/release'))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs:
+    - live_deploy
+    steps:
+    - name: Checkout source
+      uses: actions/[email protected]
+    - uses: actions/[email protected]
+      name: Download artifacts
+      with:
+        name: deploy_artifacts
+        path: wheelhouse
+    - name: Show files to release
+      shell: bash
+      run: ls -la wheelhouse
+    - run: 'echo "Automatic Release Notes. TODO: improve" > ${{ github.workspace }}-CHANGELOG.txt'
+    - name: Tag Release Commit
+      if: (startsWith(github.event.ref, 'refs/heads/release'))
+      run: |-
+        export VERSION=$(python -c "import setup; print(setup.VERSION)"
+        git tag "v$VERSION"
+        git push origin "v$VERSION"
+    - uses: softprops/action-gh-release@v1
+      name: Create Release
+      id: create_release
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        body_path: ${{ github.workspace }}-CHANGELOG.txt
+        tag_name: ${{ github.ref }}
+        release_name: Release ${{ github.ref }}
+        body: Automatic Release
+        draft: true
+        prerelease: false
+        files: |-
+          wheelhouse/*.whl
+          wheelhouse/*.asc
+          wheelhouse/*.ots
+          wheelhouse/*.zip
+          wheelhouse/*.tar.gz
 
 
 ###