Skip to content

Commit

Permalink
Reorganize the CI (#1084)
Browse files Browse the repository at this point in the history 
* Reorganize the CI
- Run on push/pull requests onto main
- Use concurrency rather than skip_duplicate
- Remove old lint job since it has been superseded by pre-commit
- Simplify CodeQL
- Upload coverage with codecov/codecov-action@v3
- Use trusted publishing
- Include test-no-myst and test-no-kernel in test-pip with matrix.include
- test quarto with a kernel
- add experimental: false and continue-on-error
- move the publish step to a dedicated workflow
  • Loading branch information
@mwouts
mwouts authored Jun 24, 2023
1 parent d7f11dc commit 468ad0d
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 168 deletions.
205 changes: 37 additions & 168 deletions .github/workflows/continuous-integration.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ on:
push:
paths-ignore:
- 'CHANGELOG.md'
branches: [main]
pull_request:
branches: [main]
schedule:
- cron: '0 11 * * 4'

Expand All @@ -12,19 +14,7 @@ concurrency:
cancel-in-progress: true

jobs:
skip_duplicate:
runs-on: ubuntu-latest
outputs:
should_skip: ${{ steps.skip_check.outputs.should_skip }}
steps:
- id: skip_check
uses: fkirc/skip-duplicate-actions@master
with:
github_token: ${{ github.token }}

pre-commit:
needs: skip_duplicate
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
Expand All @@ -34,66 +24,40 @@ jobs:
python-version: "3.x"
- uses: pre-commit/[email protected]

lint:
name: Lint
needs: skip_duplicate
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install flake8
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# all Python files should follow PEP8 (except some notebooks, see setup.cfg)
flake8 jupytext tests
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --statistics
analyse:
name: Analyse
needs: skip_duplicate
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
codeql:
needs: pre-commit
runs-on: ubuntu-latest

permissions:
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
# We must fetch at least the immediate parents so that if this is
# a pull request then we can check out the head.
fetch-depth: 2

# If this run was triggered by a pull request event, then checkout
# the head of the pull request instead of the merge commit.
- run: git checkout HEAD^2
if: ${{ github.event_name == 'pull_request' }}

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
# Override language selection
uses: github/codeql-action/init@v2
with:
languages: python, javascript

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
uses: github/codeql-action/analyze@v2

test-pip:
needs: lint
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
needs: pre-commit
strategy:
matrix:
python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11"]
markdown-it-py-version: [2]
kernel: [true]
experimental: [false]
include:
- python-version: 3.12
experimental: true
- python-version: 3.9
markdown-it-py-version: null
- python-version: 3.9
markdown-it-py-version: 3
- python-version: 3.9
kernel: false
- python-version: 3.9
quarto: true
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand All @@ -102,19 +66,26 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
continue-on-error: ${{ matrix.experimental }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
# All dependencies but markdown-it-py
python -m pip install nbformat pyyaml toml
python -m pip install -r requirements-dev.txt
# install sphinx_gallery and matplotlib if available (may not work on pypy)
# install sphinx_gallery and matplotlib if available
python -m pip install sphinx_gallery~=0.7.0 || true
python -m pip install jupyter-fs || true
- name: Install markdown-it-py
if: ${{ matrix.markdown-it-py-version }}
run: python -m pip install markdown-it-py==${{ matrix.markdown-it-py-version }}
- name: Install from source (required for the pre-commit tests)
run: python -m pip install .
run: python -m pip install . --no-deps
- name: Install a Jupyter Kernel
if: ${{ matrix.kernel }}
run: python -m ipykernel install --name python_kernel --user
- name: Install Quarto
if: ${{ matrix.quarto }}
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
run: |
Expand All @@ -125,11 +96,10 @@ jobs:
- name: Test with pytest
run: pytest --cov=./ --cov-report=xml
- name: Upload coverage
uses: codecov/codecov-action@v1
uses: codecov/codecov-action@v3

test-conda:
needs: lint
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
needs: pre-commit
strategy:
matrix:
os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ]
Expand Down Expand Up @@ -173,91 +143,11 @@ jobs:
pytest --cov=./ --cov-report=xml
}
- name: Upload coverage
uses: codecov/codecov-action@v1
uses: codecov/codecov-action@v3
if: ${{ matrix.os != 'windows-latest' }}

test-pip-no-myst:
needs: skip_duplicate
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
strategy:
matrix:
python-version: [ 3.9 ]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies, except markdown-it-py
run: |
python -m pip install --upgrade pip
# All dependencies but markdown-it-py
python -m pip install nbformat pyyaml toml
python -m pip install -r requirements-dev.txt
- name: Install a Jupyter Kernel
run: python -m ipykernel install --name python_kernel --user
- name: Test with pytest
run: pytest --cov=./ --cov-report=xml
- name: Upload coverage
uses: codecov/codecov-action@v1

test-pip-no-kernel:
needs: skip_duplicate
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
strategy:
matrix:
python-version: [ 3.9 ]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
python -m pip install -r requirements-dev.txt
- name: Test with pytest
run: pytest --cov=./ --cov-report=xml
- name: Upload coverage
uses: codecov/codecov-action@v1

test-pip-notebook-pre:
needs: skip_duplicate
if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
strategy:
matrix:
python-version: [ 3.9 ]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
# All dependencies but markdown-it-py
python -m pip install -r requirements.txt
python -m pip install -r requirements-dev.txt
# Notebook pre-release version #933
python -m pip install --pre -U notebook
- name: Install a Jupyter Kernel
run: python -m ipykernel install --name python_kernel --user
- name: Test with pytest
run: pytest --cov=./ --cov-report=xml
- name: Upload coverage
uses: codecov/codecov-action@v1

build:
name: Build
needs: test-pip
runs-on: ubuntu-latest
steps:
- name: Checkout source
Expand All @@ -280,24 +170,3 @@ jobs:
# Install
python -m pip install dist/*.tar.gz
echo "Install went OK"
publish:
name: Publish to PyPi
needs: build
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
runs-on: ubuntu-latest
steps:
- name: Checkout source
uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Build package
run: |
python -m pip install wheel jupyter-packaging 'jupyterlab>=3,<4'
BUILD_JUPYTERLAB_EXTENSION=1 python setup.py sdist bdist_wheel
- name: Publish
uses: pypa/[email protected]
with:
user: __token__
password: ${{ secrets.PYPI_KEY }}
28 changes: 28 additions & 0 deletions .github/workflows/publish.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: CI
on:
push:
tags:
- "v[0-9]+.[0-9]+.[0-9]+"
- "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"

jobs:
publish:
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/p/jupytext
permissions:
id-token: write
steps:
- name: Checkout source
uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Build package
run: |
python -m pip install wheel jupyter-packaging 'jupyterlab>=3,<4'
BUILD_JUPYTERLAB_EXTENSION=1 python setup.py sdist bdist_wheel
- name: Publish
uses: pypa/gh-action-pypi-publish@release/v1
3 changes: 3 additions & 0 deletions docs/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ Jupytext ChangeLog
1.14.7-dev (2023-06-??)
-------------------

**Changed**
- We have updated the GitHub workflows - thanks to Matthew Feickert and to Cristian Le for their help on this subject ([#1037](https://github.com/mwouts/jupytext/issues/1037))

**Fixed**
- Notebooks with an empty YAML header work ([#1070](https://github.com/mwouts/jupytext/issues/1070))
- Double quote strings in R Markdown options can contain single quotes ([#1079](https://github.com/mwouts/jupytext/issues/1079))
Expand Down

0 comments on commit 468ad0d

Please sign in to comment.