Skip to content

Commit

Permalink
Add nightly workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
cjdsellers committed Jan 6, 2024
1 parent af2fd59 commit 9c8af5d
Show file tree
Hide file tree
Showing 13 changed files with 194 additions and 39 deletions.
4 changes: 1 addition & 3 deletions .github/workflows/build-wheels.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
name: build-wheels

# Build wheels on every push to `develop` branch

on:
push:
branches: [develop]
branches: [nightly]

jobs:
build-wheels:
Expand Down
4 changes: 1 addition & 3 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
name: build

# Build and test NautilusTrader

on:
push:
branches: [master, develop]
Expand All @@ -14,7 +12,7 @@ jobs:
fail-fast: false
matrix:
arch: [x64]
os: [ubuntu-latest, macos-latest, windows-latest]
os: [ubuntu-latest, windows-latest]
python-version: ["3.10", "3.11", "3.12"]
defaults:
run:
Expand Down
4 changes: 1 addition & 3 deletions .github/workflows/coverage.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
name: coverage

# Run code coverage analysis for NautilusTrader

on:
push:
branches: [develop]
Expand All @@ -13,7 +11,7 @@ jobs:
matrix:
arch: [x64]
os: [ubuntu-latest]
python-version: ["3.10"]
python-version: ["3.11"]
name: build - Python ${{ matrix.python-version }} (${{ matrix.arch }} ${{ matrix.os }})
runs-on: ${{ matrix.os }}

Expand Down
24 changes: 11 additions & 13 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
name: docker

# Build NautilusTrader docker images

on:
push:
branches: [master, develop]
branches: [master, nightly]

jobs:
build-docker-images:
Expand Down Expand Up @@ -47,18 +45,18 @@ jobs:
id: branch-name
uses: tj-actions/[email protected]

- name: Build nautilus_trader image (develop)
if: ${{ steps.branch-name.outputs.current_branch == 'develop' }}
id: docker_build_trader_develop
- name: Build nautilus_trader image (nightly)
if: ${{ steps.branch-name.outputs.current_branch == 'nightly' }}
id: docker_build_trader_nightly
uses: docker/build-push-action@v5
with:
file: ".docker/nautilus_trader.dockerfile"
push: true
tags: ghcr.io/${{ github.repository_owner }}/nautilus_trader:develop
tags: ghcr.io/${{ github.repository_owner }}/nautilus_trader:nightly
cache-from: type=gha
cache-to: type=gha
- name: Digest nautilus_trader image
run: echo ${{ steps.docker_build_trader_develop.outputs.digest }}
run: echo ${{ steps.docker_build_trader_nightly.outputs.digest }}

- name: Build nautilus_trader image (latest)
if: ${{ steps.branch-name.outputs.current_branch == 'master' }}
Expand All @@ -73,20 +71,20 @@ jobs:
- name: Digest nautilus_trader image
run: echo ${{ steps.docker_build_trader_latest.outputs.digest }}

- name: Build jupyterlab image (develop)
if: ${{ steps.branch-name.outputs.current_branch == 'develop' }}
id: docker_build_jupyterlab_develop
- name: Build jupyterlab image (nightly)
if: ${{ steps.branch-name.outputs.current_branch == 'nightly' }}
id: docker_build_jupyterlab_nightly
uses: docker/build-push-action@v5
with:
file: ".docker/jupyterlab.dockerfile"
push: true
tags: ghcr.io/${{ github.repository_owner }}/jupyterlab:develop
tags: ghcr.io/${{ github.repository_owner }}/jupyterlab:nightly
cache-from: type=gha
cache-to: type=gha
build-args: |
GIT_TAG=${{ steps.branch-name.outputs.current_branch }}
- name: Digest jupyterlab image
run: echo ${{ steps.docker_build_jupyterlab_develop.outputs.digest }}
run: echo ${{ steps.docker_build_jupyterlab_nightly.outputs.digest }}

- name: Build jupyterlab image (latest)
if: ${{ steps.branch-name.outputs.current_branch == 'master' }}
Expand Down
10 changes: 4 additions & 6 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
name: docs

# Build and deploy the NautilusTrader documentation

on:
push:
branches: [master, develop]
branches: [nightly]

jobs:
docs:
Expand Down Expand Up @@ -75,16 +73,16 @@ jobs:
- name: Add CNAME
run: echo "docs.nautilustrader.io" >> docs/build/html/CNAME

- name: Publish docs (develop)
if: ${{ steps.branch-name.outputs.current_branch == 'develop' }}
- name: Publish docs (nightly)
if: ${{ steps.branch-name.outputs.current_branch == 'nightly' }}
uses: s0/git-publish-subdir-action@develop
env:
REPO: self
BRANCH: gh-pages
FOLDER: docs/build/html
SQUASH_HISTORY: false
GITHUB_TOKEN: ${{ secrets.GHPAGES_ACCESS }}
TARGET_DIR: develop
TARGET_DIR: nightly

- name: Publish docs (latest)
if: ${{ steps.branch-name.outputs.current_branch == 'master' }}
Expand Down
25 changes: 25 additions & 0 deletions .github/workflows/nightly-merge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: nightly-merge

on:
push:
branches: [nightly-merge-test]
schedule:
- cron: '0 14 * * *' # At 14:00 UTC every day

jobs:
nightly-merge:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all branches and tags

- name: Merge develop into nightly
run: |
git checkout nightly
git merge --no-ff origin/develop -m "Automated merge of develop into nightly"
- name: Push changes
run: |
git push origin nightly
129 changes: 129 additions & 0 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
name: nightly

on:
push:
branches: [nightly]

jobs:
build:
strategy:
fail-fast: false
matrix:
arch: [x64]
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ["3.10", "3.11", "3.12"]
defaults:
run:
shell: bash
name: build - Python ${{ matrix.python-version }} (${{ matrix.arch }} ${{ matrix.os }})
runs-on: ${{ matrix.os }}
env:
BUILD_MODE: debug
RUST_BACKTRACE: 1

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Get Rust version from rust-toolchain.toml
id: rust-version
run: |
version=$(awk -F\" '/version/ {print $2}' nautilus_core/rust-toolchain.toml)
echo "Rust toolchain version $version"
echo "RUST_VERSION=$version" >> $GITHUB_ENV
working-directory: ${{ github.workspace }}

- name: Set up Rust tool-chain (Linux, Windows) stable
if: (runner.os == 'Linux') || (runner.os == 'Windows')
uses: actions-rust-lang/[email protected]
with:
toolchain: ${{ env.RUST_VERSION }}
components: rustfmt, clippy

# Work around as actions-rust-lang does not seem to work on macOS yet
- name: Set up Rust tool-chain (macOS) stable
if: runner.os == 'macOS'
uses: actions-rs/toolchain@v1
with:
toolchain: ${{ env.RUST_VERSION }}
override: true
components: rustfmt, clippy

- name: Set up Python environment
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

- name: Get Poetry version from poetry-version
run: |
version=$(cat poetry-version)
echo "POETRY_VERSION=$version" >> $GITHUB_ENV
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: ${{ env.POETRY_VERSION }}

- name: Install build dependencies
run: python -m pip install --upgrade pip setuptools wheel pre-commit msgspec

- name: Install TA-Lib (Linux)
if: runner.os == 'Linux'
run: |
make install-talib
poetry run pip install ta-lib
- name: Setup cached pre-commit
id: cached-pre-commit
uses: actions/cache@v3
with:
path: ~/.cache/pre-commit
key: ${{ runner.os }}-${{ matrix.python-version }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}

- name: Set poetry cache-dir
run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV

- name: Poetry cache
id: cached-poetry
uses: actions/cache@v3
with:
path: ${{ env.POETRY_CACHE_DIR }}
key: ${{ runner.os }}-${{ matrix.python-version }}-poetry-${{ hashFiles('**/poetry.lock') }}

- name: Run pre-commit
run: |
# pre-commit run --hook-stage manual gitlint-ci
pre-commit run --all-files
- name: Install Redis (macOS)
if: runner.os == 'macOS'
run: |
brew install redis
redis-server --daemonize yes
- name: Install Redis (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get install redis-server
redis-server --daemonize yes
- name: Run nautilus_core cargo tests (Linux, macOS)
if: (runner.os == 'Linux') || (runner.os == 'macOS')
run: |
cargo install cargo-nextest
make cargo-test
- name: Run tests (Linux, macOS)
if: (runner.os == 'Linux') || (runner.os == 'macOS')
run: |
make pytest
make test-examples
# Run tests without parallel build (avoids linker errors)
- name: Run tests (Windows)
if: runner.os == 'Windows'
run: |
poetry install --with test --all-extras
poetry run pytest --ignore=tests/performance_tests --new-first --failed-first
env:
PARALLEL_BUILD: false
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ cargo-doc:

.PHONY: docker-build
docker-build: clean
docker pull ${IMAGE_FULL} || docker pull ${IMAGE}:develop || true
docker pull ${IMAGE_FULL} || docker pull ${IMAGE}:nightly || true
docker build -f .docker/nautilus_trader.dockerfile --platform linux/x86_64 -t ${IMAGE_FULL} .

.PHONY: docker-build-force
Expand Down
16 changes: 12 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
| Branch | Version | Status |
| :-------- | :-------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `master` | ![version](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fnautechsystems%2Fnautilus_trader%2Fmaster%2Fversion.json) | [![build](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml) |
| `nightly` | ![version](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fnautechsystems%2Fnautilus_trader%2Fnightly%2Fversion.json) | [![build](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml/badge.svg?branch=nightly)](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml) |
| `develop` | ![version](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fnautechsystems%2Fnautilus_trader%2Fdevelop%2Fversion.json) | [![build](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/nautechsystems/nautilus_trader/actions/workflows/build.yml) |

| Platform | Rust | Python |
Expand Down Expand Up @@ -215,6 +216,7 @@ Documentation of these changes in the release notes are made on a best-effort ba
### Branches
- `master` branch will always reflect the source code for the latest released version
- `nightly` branch is automatically merged from `develop` branch daily at 14:00 UTC, or when required
- `develop` branch is normally very active with frequent commits and may contain experimental features. We aim to maintain a stable
passing build on this branch
Expand Down Expand Up @@ -254,8 +256,10 @@ The below are some examples of this:
Docker containers are built using a base `python:3.11-slim` with the following image variant tags:
- `nautilus_trader:latest` has the latest release version installed
- `nautilus_trader:develop` has the head of the `develop` branch installed
- `jupyterlab:develop` has the head of the `develop` branch installed along with `jupyterlab` and an
- `nautilus_trader:nightly` has the head of the `nightly` branch installed
- `jupyterlab:latest` has the head of the `master` branch installed along with `jupyterlab` and an
example backtest notebook with accompanying data
- `jupyterlab:nightly` has the head of the `nightly` branch installed along with `jupyterlab` and an
example backtest notebook with accompanying data
The container images can be pulled as follows:
Expand All @@ -264,8 +268,8 @@ The container images can be pulled as follows:
You can launch the backtest example container by running:
docker pull ghcr.io/nautechsystems/jupyterlab:develop --platform linux/amd64
docker run -p 8888:8888 ghcr.io/nautechsystems/jupyterlab:develop
docker pull ghcr.io/nautechsystems/jupyterlab:nightly --platform linux/amd64
docker run -p 8888:8888 ghcr.io/nautechsystems/jupyterlab:nightly
Then open your browser at the following address:
Expand Down Expand Up @@ -400,6 +404,10 @@ class EMACross(Strategy):
We aim to provide the most pleasant developer experience possible for this hybrid codebase of Python, Cython and Rust.
Refer to the [Developer Guide](https://docs.nautilustrader.io/developer_guide/index.html) for helpful information.

`cargo-nextest` is the standard Rust test runner for NautilusTrader. You can install it by running:

cargo install cargo-nextest

## Contributing

Thank you for considering contributing to Nautilus Trader! We welcome any and all help to improve
Expand Down
5 changes: 4 additions & 1 deletion docs/_static/version.json
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
{"latest": "", "develop": "develop"}
{
"latest": "",
"nightly": "nightly"
}
2 changes: 1 addition & 1 deletion docs/api_reference/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ from the latest NautilusTrader source code using [Sphinx](https://www.sphinx-doc
Please note that there are separate references for different versions of NautilusTrader:

- **Latest**: This API reference is built from the head of the `master` branch and represents the latest stable release.
- **Develop**: This API reference is built from the head of the `develop` branch and represents bleeding edge and experimental changes/features currently in development.
- **Nightly**: This API reference is built from the head of the `nightly` branch and represents bleeding edge and experimental changes/features currently in development.

You can select the desired API reference from the **Versions** top right drop down menu.

Expand Down
4 changes: 2 additions & 2 deletions docs/getting_started/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ deleted when the container is deleted.
- To get started, install docker:
- Go to [docker.com](https://docs.docker.com/get-docker/) and follow the instructions
- From a terminal, download the latest image
- `docker pull ghcr.io/nautechsystems/jupyterlab:develop --platform linux/amd64`
- `docker pull ghcr.io/nautechsystems/jupyterlab:nightly --platform linux/amd64`
- Run the docker container, exposing the jupyter port:
- `docker run -p 8888:8888 ghcr.io/nautechsystems/jupyterlab:develop`
- `docker run -p 8888:8888 ghcr.io/nautechsystems/jupyterlab:nightly`
- Open your web browser to `localhost:{port}`
- https://localhost:8888

Expand Down
4 changes: 2 additions & 2 deletions docs/rust.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ Use the following links to explore the Rust docs API references for two differen
## [Latest Rust docs](https://docs.nautilustrader.io/core)
This API reference is built from the HEAD of the `master` branch and represents the latest stable release.

## [Develop Rust docs](https://docs.nautilustrader.io/develop/core)
This API reference is built from the HEAD of the `develop` branch and represents bleeding edge and experimental changes/features currently in development.
## [Nightly Rust docs](https://docs.nautilustrader.io/nightly/core)
This API reference is built from the HEAD of the `nightly` branch and represents bleeding edge and experimental changes/features currently in development.

## What is Rust?
[Rust](https://www.rust-lang.org/) is a multi-paradigm programming language designed for performance and safety, especially safe
Expand Down

0 comments on commit 9c8af5d

Please sign in to comment.