Estimation Enhancements #836

Workflow file for this run

.github/workflows/core_tests.yml at 420ed8e

	name: Core Testing

	on:
	push:
	branches:
	- '*'

	pull_request:
	branches:
	- '*'

	env:
	CACHE_NUMBER: 2 # increase to reset cache manually

	jobs:
	foundation:

	strategy:
	matrix:
	python-version: ["3.10"]
	defaults:
	run:
	shell: bash -l {0}
	name: linux-64-py${{ matrix.python-version }}
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ matrix.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: /usr/share/miniconda3/envs/asim-test
	key: linux-64-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install -e . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	- name: Lint with Black
	run: \|
	# stop the build if there are problems
	black --check --diff .

	- name: Test activitysim.core
	run: \|
	python -m pytest --pyargs activitysim.core

	- name: Test activitysim.abm.models
	run: \|
	python -m pytest --pyargs activitysim.abm.models

	- name: Test activitysim.abm.test
	run: \|
	python -m pytest --pyargs activitysim.abm.test

	- name: Test activitysim.cli
	run: \|
	python -m pytest --pyargs activitysim.cli

	- name: Test activitysim.examples.test
	run: \|
	python -m pytest --pyargs activitysim.examples.test


	cross-platform:
	# also test foundation cross platforms, but do not require a successful
	# completion before starting regional model tests
	needs: foundation
	strategy:
	matrix:
	include:
	- os: macos-latest
	label: macOS
	python-version: "3.10"

	- os: windows-latest
	label: win-64
	python-version: "3.10"

	defaults:
	run:
	shell: bash -l {0}

	name: ${{ matrix.label }}
	runs-on: ${{ matrix.os }}
	steps:
	- uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ matrix.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: ${{ env.CONDA }}/envs
	key: ${{ matrix.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install -e . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	- name: Lint with Black
	run: \|
	# stop the build if there are problems
	black --check --diff .

	- name: Test activitysim.core
	run: \|
	python -m pytest --pyargs activitysim.core

	- name: Test activitysim.abm.models
	run: \|
	python -m pytest --pyargs activitysim.abm.models

	- name: Test activitysim.abm.test
	run: \|
	python -m pytest --pyargs activitysim.abm.test

	- name: Test activitysim.cli
	run: \|
	python -m pytest --pyargs activitysim.cli


	builtin_regional_models:
	needs: foundation
	env:
	python-version: "3.10"
	label: linux-64
	strategy:
	matrix:
	region:
	- prototype_mtc
	- placeholder_psrc
	- prototype_marin
	- prototype_mtc_extended
	- placeholder_sandag
	- prototype_sandag_xborder
	- production_semcog
	- prototype_mwcog
	- placeholder_multiple_zone
	fail-fast: false
	defaults:
	run:
	shell: bash -l {0}
	name: ${{ matrix.region }}
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ env.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: ${{ env.CONDA }}/envs
	key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install -e . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	# TODO: Cache sharrow compiled flows? The contents of __pycache__ appear to
	# be ignored, so this is not working as expected right now
	#
	# - name: Define Inputs
	# run: echo "REGION_DEFS=activitysim/examples/${{ matrix.region }}/** " >> $GITHUB_ENV
	#
	# - name: Get a random number
	# run: echo "RANDOM_SUFFIX=${RANDOM}${RANDOM}" >> $GITHUB_ENV
	#
	# - uses: actions/cache@v3
	# # store the regional model's cache directory in github actions cache
	# # this will (almost) never hit on primary key due to the random number
	# # but will pull the most recent cache from restore-keys... and then
	# # update the cache with additional compiled flows as needed.
	# # Hoping this will result in fewer re-compiles on tests and faster
	# # testing overall
	# with:
	# path: activitysim/examples/${{ matrix.region }}/test/output/cache
	# key: ${{ matrix.region }}-${{ env.label }}-${{ hashFiles(env.REGION_DEFS) }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}-${{ env.RANDOM_SUFFIX }}
	# restore-keys: \|
	# ${{ matrix.region }}-${{ env.label }}-${{ hashFiles(env.REGION_DEFS) }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	# id: cache-example

	- name: Test ${{ matrix.region }}
	run: \|
	python -m pytest activitysim/examples/${{ matrix.region }}/test --durations=0

	external_regional_models:
	needs: foundation
	env:
	python-version: "3.10"
	label: linux-64
	strategy:
	matrix:
	include:
	- region: Standard 1-Zone Example (MTC)
	region-org: ActivitySim
	region-repo: activitysim-prototype-mtc
	region-branch: extended
	- region: Standard 2-Zone Example (SANDAG)
	region-org: ActivitySim
	region-repo: sandag-abm3-example
	region-branch: main
	fail-fast: false
	defaults:
	run:
	shell: bash -l {0}
	name: ${{ matrix.region }}
	runs-on: ubuntu-latest
	steps:
	- name: Checkout ActivitySim
	uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ env.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: \|
	${{ env.CONDA }}/envs
	~/.cache/ActivitySim
	key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	- name: Checkout Example
	uses: actions/checkout@v4
	with:
	repository: '${{ matrix.region-org }}/${{ matrix.region-repo }}'
	ref: '${{ matrix.region-branch }}'
	path: '${{ matrix.region-repo }}'

	- name: Test ${{ matrix.region }}
	run: \|
	cd ${{ matrix.region-repo }}/test
	python -m pytest .

	random_seed_generation:
	needs: foundation
	env:
	python-version: "3.10"
	label: linux-64
	defaults:
	run:
	shell: bash -l {0}
	name: random_seed_generation_test
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ env.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: ${{ env.CONDA }}/envs
	key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install -e . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	- name: Test Random Seed Generation
	run: \|
	python -m pytest test/random_seed/test_random_seed.py --durations=0

	estimation_mode:
	needs: foundation
	env:
	python-version: "3.10"
	label: linux-64
	defaults:
	run:
	shell: bash -l {0}
	name: Estimation Mode Unit Tests
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ env.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: ${{ env.CONDA }}/envs
	key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install Larch v6
	run: python -m pip install larch6

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install -e . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	- name: Test Estimation Mode
	run: \|
	python -m pytest activitysim/estimation/test/test_larch_estimation.py --durations=0

	estimation_notebooks:
	needs: foundation
	env:
	python-version: "3.10"
	label: linux-64
	defaults:
	run:
	shell: bash -l {0}
	name: Estimation Notebooks Test
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ env.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: ${{ env.CONDA }}/envs
	key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install Graphviz
	uses: ts-graphviz/setup-graphviz@v2

	- name: Install Larch v6
	run: python -m pip install larch6 "pandas<2" pydot

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install -e . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	- name: Create Estimation Data
	run: \|
	python activitysim/examples/example_estimation/notebooks/est_mode_setup.py --household_sample_size 5000

	- name: Test Estimation Notebooks
	run: \|
	python -m pytest activitysim/examples/example_estimation/notebooks/*.ipynb \
	--nbmake-timeout=3000 \
	--ignore=activitysim/examples/example_estimation/notebooks/01_estimation_mode.ipynb

	estimation_edb_creation:
	needs: foundation
	env:
	python-version: "3.10"
	label: linux-64
	defaults:
	run:
	shell: bash -l {0}
	name: estimation_edb_creation_test
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4

	- name: Setup Miniforge
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	activate-environment: asim-test
	python-version: ${{ env.python-version }}

	- name: Set cache date for year and month
	run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

	- uses: actions/cache@v4
	with:
	path: ${{ env.CONDA }}/envs
	key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
	id: cache

	- name: Update environment
	run: \|
	conda env update -n asim-test -f conda-environments/github-actions-tests.yml
	if: steps.cache.outputs.cache-hit != 'true'

	- name: Install activitysim
	# installing without dependencies is faster, we trust that all needed dependencies
	# are in the conda environment defined above. Also, this avoids pip getting
	# confused and reinstalling tables (pytables).
	run: \|
	python -m pip install -e . --no-deps

	- name: Conda checkup
	run: \|
	conda info -a
	conda list

	- name: Test Estimation EDB Creation
	run: \|
	python -m pytest activitysim/estimation/test/test_edb_creation/test_edb_formation.py --durations=0

	develop-docbuild:
	needs: foundation
	if: github.ref_name == 'main'
	name: develop-docbuild
	runs-on: ubuntu-latest
	defaults:
	run:
	shell: bash -l {0}
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # get all tags, lets setuptools_scm do its thing
	- name: Set up Python 3.10
	uses: actions/setup-python@v5
	with:
	python-version: "3.10"
	- name: Install dependencies
	uses: conda-incubator/setup-miniconda@v3
	with:
	miniforge-version: latest
	environment-file: conda-environments/docbuild.yml
	python-version: "3.10"
	activate-environment: docbuild
	auto-activate-base: false
	auto-update-conda: false
	- name: Install activitysim
	run: \|
	python -m pip install .
	- name: Conda checkup
	run: \|
	conda info -a
	conda list
	echo ${{ github.repository }}
	echo ${{ github.ref_name }}
	- name: localize version switcher
	run: \|
	python .github/workflows/localize-base-urls.py docs/_static/switcher.json
	git update-index --assume-unchanged docs/_static/switcher.json
	cat docs/_static/switcher.json
	- name: Build the docs
	run: \|
	cd docs
	make clean
	make html
	- name: Push to GitHub Pages
	uses: peaceiris/[email protected]
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	# Token is created automatically by Github Actions, no other config needed
	publish_dir: ./docs/_build/html
	destination_dir: develop

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Estimation Enhancements #836

Workflow file

Estimation Enhancements #836

Jobs

Run details

Workflow file for this run