From 468ad0d7a56e1e9341c08608a25707633688ee04 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Sat, 24 Jun 2023 18:32:25 +0100 Subject: [PATCH] Reorganize the CI (#1084) * Reorganize the CI - Run on push/pull requests onto main - Use concurrency rather than skip_duplicate - Remove old lint job since it has been superseded by pre-commit - Simplify CodeQL - Upload coverage with codecov/codecov-action@v3 - Use trusted publishing - Include test-no-myst and test-no-kernel in test-pip with matrix.include - test quarto with a kernel - add experimental: false and continue-on-error - move the publish step to a dedicated workflow --- .github/workflows/continuous-integration.yml | 205 ++++--------------- .github/workflows/publish.yml | 28 +++ docs/CHANGELOG.md | 3 + 3 files changed, 68 insertions(+), 168 deletions(-) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 06062edcb..4c1f1b6f5 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,7 +3,9 @@ on: push: paths-ignore: - 'CHANGELOG.md' + branches: [main] pull_request: + branches: [main] schedule: - cron: '0 11 * * 4' @@ -12,19 +14,7 @@ concurrency: cancel-in-progress: true jobs: - skip_duplicate: - runs-on: ubuntu-latest - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@master - with: - github_token: ${{ github.token }} - pre-commit: - needs: skip_duplicate - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -34,66 +24,40 @@ jobs: python-version: "3.x" - uses: pre-commit/action@v3.0.0 - lint: - name: Lint - needs: skip_duplicate - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # all Python files should follow PEP8 (except some notebooks, see setup.cfg) - flake8 jupytext tests - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --statistics - - analyse: - name: Analyse - needs: skip_duplicate - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} + codeql: + needs: pre-commit runs-on: ubuntu-latest - + permissions: + security-events: write steps: - name: Checkout repository uses: actions/checkout@v3 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can check out the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - # Override language selection + uses: github/codeql-action/init@v2 with: languages: python, javascript - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 test-pip: - needs: lint - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} + needs: pre-commit strategy: matrix: python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11"] + markdown-it-py-version: [2] + kernel: [true] + experimental: [false] + include: + - python-version: 3.12 + experimental: true + - python-version: 3.9 + markdown-it-py-version: null + - python-version: 3.9 + markdown-it-py-version: 3 + - python-version: 3.9 + kernel: false + - python-version: 3.9 + quarto: true runs-on: ubuntu-latest steps: - name: Checkout @@ -102,19 +66,26 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + continue-on-error: ${{ matrix.experimental }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements.txt + # All dependencies but markdown-it-py + python -m pip install nbformat pyyaml toml python -m pip install -r requirements-dev.txt - # install sphinx_gallery and matplotlib if available (may not work on pypy) + # install sphinx_gallery and matplotlib if available python -m pip install sphinx_gallery~=0.7.0 || true python -m pip install jupyter-fs || true + - name: Install markdown-it-py + if: ${{ matrix.markdown-it-py-version }} + run: python -m pip install markdown-it-py==${{ matrix.markdown-it-py-version }} - name: Install from source (required for the pre-commit tests) - run: python -m pip install . + run: python -m pip install . --no-deps - name: Install a Jupyter Kernel + if: ${{ matrix.kernel }} run: python -m ipykernel install --name python_kernel --user - name: Install Quarto + if: ${{ matrix.quarto }} env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} run: | @@ -125,11 +96,10 @@ jobs: - name: Test with pytest run: pytest --cov=./ --cov-report=xml - name: Upload coverage - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 test-conda: - needs: lint - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} + needs: pre-commit strategy: matrix: os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ] @@ -173,91 +143,11 @@ jobs: pytest --cov=./ --cov-report=xml } - name: Upload coverage - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 if: ${{ matrix.os != 'windows-latest' }} - test-pip-no-myst: - needs: skip_duplicate - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} - strategy: - matrix: - python-version: [ 3.9 ] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies, except markdown-it-py - run: | - python -m pip install --upgrade pip - # All dependencies but markdown-it-py - python -m pip install nbformat pyyaml toml - python -m pip install -r requirements-dev.txt - - name: Install a Jupyter Kernel - run: python -m ipykernel install --name python_kernel --user - - name: Test with pytest - run: pytest --cov=./ --cov-report=xml - - name: Upload coverage - uses: codecov/codecov-action@v1 - - test-pip-no-kernel: - needs: skip_duplicate - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} - strategy: - matrix: - python-version: [ 3.9 ] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - python -m pip install -r requirements-dev.txt - - name: Test with pytest - run: pytest --cov=./ --cov-report=xml - - name: Upload coverage - uses: codecov/codecov-action@v1 - - test-pip-notebook-pre: - needs: skip_duplicate - if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }} - strategy: - matrix: - python-version: [ 3.9 ] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # All dependencies but markdown-it-py - python -m pip install -r requirements.txt - python -m pip install -r requirements-dev.txt - # Notebook pre-release version #933 - python -m pip install --pre -U notebook - - name: Install a Jupyter Kernel - run: python -m ipykernel install --name python_kernel --user - - name: Test with pytest - run: pytest --cov=./ --cov-report=xml - - name: Upload coverage - uses: codecov/codecov-action@v1 - build: - name: Build + needs: test-pip runs-on: ubuntu-latest steps: - name: Checkout source @@ -280,24 +170,3 @@ jobs: # Install python -m pip install dist/*.tar.gz echo "Install went OK" - publish: - name: Publish to PyPi - needs: build - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Build package - run: | - python -m pip install wheel jupyter-packaging 'jupyterlab>=3,<4' - BUILD_JUPYTERLAB_EXTENSION=1 python setup.py sdist bdist_wheel - - name: Publish - uses: pypa/gh-action-pypi-publish@v1.1.0 - with: - user: __token__ - password: ${{ secrets.PYPI_KEY }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..1a042a359 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,28 @@ +name: CI +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" + +jobs: + publish: + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/jupytext + permissions: + id-token: write + steps: + - name: Checkout source + uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Build package + run: | + python -m pip install wheel jupyter-packaging 'jupyterlab>=3,<4' + BUILD_JUPYTERLAB_EXTENSION=1 python setup.py sdist bdist_wheel + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 140ea5860..1a8fdd9c6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,9 @@ Jupytext ChangeLog 1.14.7-dev (2023-06-??) ------------------- +**Changed** +- We have updated the GitHub workflows - thanks to Matthew Feickert and to Cristian Le for their help on this subject ([#1037](https://github.com/mwouts/jupytext/issues/1037)) + **Fixed** - Notebooks with an empty YAML header work ([#1070](https://github.com/mwouts/jupytext/issues/1070)) - Double quote strings in R Markdown options can contain single quotes ([#1079](https://github.com/mwouts/jupytext/issues/1079))