Skip to content
Core Testing

Explicit chunking #469

Sign in to view logs

Explicit chunking

Explicit chunking #469

Workflow file for this run

.github/workflows/core_tests.yml at 3c047e0
name: Core Testing
on:
push:
branches:
- '*'
pull_request:
branches:
- '*'
env:
CACHE_NUMBER: 0 # increase to reset cache manually
jobs:
foundation:
strategy:
matrix:
python-version: ["3.10"]
defaults:
run:
shell: bash -l {0}
name: linux-64-py${{ matrix.python-version }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
activate-environment: asim-test
use-mamba: true
python-version: ${{ matrix.python-version }}
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
- uses: actions/cache@v3
with:
path: /usr/share/miniconda3/envs/asim-test
key: linux-64-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache
- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=1.10.13" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'
- name: Install activitysim
# installing without dependencies is faster, we trust that all needed dependencies
# are in the conda environment defined above. Also, this avoids pip getting
# confused and reinstalling tables (pytables).
run: |
python -m pip install -e . --no-deps
- name: Conda checkup
run: |
mamba info -a
mamba list
- name: Lint with Black
run: |
# stop the build if there are problems
black --check --diff .
- name: Test activitysim.core
run: |
python -m pytest --pyargs activitysim.core
- name: Test activitysim.abm.models
run: |
python -m pytest --pyargs activitysim.abm.models
- name: Test activitysim.abm.test
run: |
python -m pytest --pyargs activitysim.abm.test
- name: Test activitysim.cli
run: |
python -m pytest --pyargs activitysim.cli
- name: Test activitysim.examples.test
run: |
python -m pytest --pyargs activitysim.examples.test
cross-platform:
# also test foundation cross platforms, but do not require a successful
# completion before starting regional model tests
needs: foundation
strategy:
matrix:
include:
- os: macos-latest
label: macOS
prefix: /Users/runner/miniconda3/envs/asim-test
python-version: "3.10"
- os: windows-latest
label: win-64
prefix: C:\Miniconda3\envs\asim-test
python-version: "3.10"
defaults:
run:
shell: bash -l {0}
name: ${{ matrix.label }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
activate-environment: asim-test
use-mamba: true
python-version: ${{ matrix.python-version }}
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
- uses: actions/cache@v3
with:
path: ${{ matrix.prefix }}
key: ${{ matrix.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache
- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=1.10.13" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'
- name: Install activitysim
# installing without dependencies is faster, we trust that all needed dependencies
# are in the conda environment defined above. Also, this avoids pip getting
# confused and reinstalling tables (pytables).
run: |
python -m pip install -e . --no-deps
- name: Conda checkup
run: |
mamba info -a
mamba list
- name: Lint with Black
run: |
# stop the build if there are problems
black --check --diff .
- name: Test activitysim.core
run: |
python -m pytest --pyargs activitysim.core
- name: Test activitysim.abm.models
run: |
python -m pytest --pyargs activitysim.abm.models
- name: Test activitysim.abm.test
run: |
python -m pytest --pyargs activitysim.abm.test
- name: Test activitysim.cli
run: |
python -m pytest --pyargs activitysim.cli
builtin_regional_models:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
strategy:
matrix:
region:
- prototype_mtc
- prototype_arc
- placeholder_psrc
- prototype_marin
- prototype_mtc_extended
- placeholder_sandag
- prototype_sandag_xborder
- production_semcog
- prototype_mwcog
- placeholder_multiple_zone
fail-fast: false
defaults:
run:
shell: bash -l {0}
name: ${{ matrix.region }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
activate-environment: asim-test
use-mamba: true
python-version: ${{ env.python-version }}
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
- uses: actions/cache@v3
with:
path: ${{ env.mamba-env-prefix }}
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache
- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=1.10.13" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'
- name: Install activitysim
# installing without dependencies is faster, we trust that all needed dependencies
# are in the conda environment defined above. Also, this avoids pip getting
# confused and reinstalling tables (pytables).
run: |
python -m pip install -e . --no-deps
- name: Conda checkup
run: |
mamba info -a
mamba list
# TODO: Cache sharrow compiled flows? The contents of __pycache__ appear to
# be ignored, so this is not working as expected right now
#
# - name: Define Inputs
# run: echo "REGION_DEFS=activitysim/examples/${{ matrix.region }}/** " >> $GITHUB_ENV
#
# - name: Get a random number
# run: echo "RANDOM_SUFFIX=${RANDOM}${RANDOM}" >> $GITHUB_ENV
#
# - uses: actions/cache@v3
# # store the regional model's cache directory in github actions cache
# # this will (almost) never hit on primary key due to the random number
# # but will pull the most recent cache from restore-keys... and then
# # update the cache with additional compiled flows as needed.
# # Hoping this will result in fewer re-compiles on tests and faster
# # testing overall
# with:
# path: activitysim/examples/${{ matrix.region }}/test/output/cache
# key: ${{ matrix.region }}-${{ env.label }}-${{ hashFiles(env.REGION_DEFS) }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}-${{ env.RANDOM_SUFFIX }}
# restore-keys: |
# ${{ matrix.region }}-${{ env.label }}-${{ hashFiles(env.REGION_DEFS) }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
# id: cache-example
- name: Test ${{ matrix.region }}
run: |
python -m pytest activitysim/examples/${{ matrix.region }}/test --durations=0
external_regional_models:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
strategy:
matrix:
region:
- prototype_mtc
- prototype_psrc_in_development
fail-fast: false
defaults:
run:
shell: bash -l {0}
name: ${{ matrix.region }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
activate-environment: asim-test
use-mamba: true
python-version: ${{ env.python-version }}
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
- uses: actions/cache@v3
with:
path: |
${{ env.mamba-env-prefix }}
~/.cache/ActivitySim
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache
- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=1.10.13" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'
- name: Install activitysim
# installing without dependencies is faster, we trust that all needed dependencies
# are in the conda environment defined above. Also, this avoids pip getting
# confused and reinstalling tables (pytables).
run: |
python -m pip install -e . --no-deps
- name: Conda checkup
run: |
mamba info -a
mamba list
- name: Test ${{ matrix.region }}
run: |
python -m activitysim test ${{ matrix.region }}
random_seed_generation:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
defaults:
run:
shell: bash -l {0}
name: random_seed_generation_test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
activate-environment: asim-test
use-mamba: true
python-version: ${{ env.python-version }}
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
- uses: actions/cache@v3
with:
path: ${{ env.mamba-env-prefix }}
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache
- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=1.10.13" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'
- name: Install activitysim
# installing without dependencies is faster, we trust that all needed dependencies
# are in the conda environment defined above. Also, this avoids pip getting
# confused and reinstalling tables (pytables).
run: |
python -m pip install -e . --no-deps
- name: Conda checkup
run: |
mamba info -a
mamba list
- name: Test Random Seed Generation
run: |
python -m pytest test/random_seed/test_random_seed.py --durations=0
estimation_mode:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
defaults:
run:
shell: bash -l {0}
name: estimation_mode_test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
activate-environment: asim-test
use-mamba: true
python-version: ${{ env.python-version }}
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
- uses: actions/cache@v3
with:
path: ${{ env.mamba-env-prefix }}
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache
- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=1.10.13" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'
- name: Install Larch
run: mamba install "larch>=5.7.1"
- name: Install activitysim
# installing without dependencies is faster, we trust that all needed dependencies
# are in the conda environment defined above. Also, this avoids pip getting
# confused and reinstalling tables (pytables).
run: |
python -m pip install -e . --no-deps
- name: Conda checkup
run: |
mamba info -a
mamba list
- name: Test Estimation Mode
run: |
python -m pytest activitysim/estimation/test/test_larch_estimation.py --durations=0
develop-docbuild:
needs: foundation
if: github.ref_name == 'develop'
name: develop-docbuild
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # get all tags, lets setuptools_scm do its thing
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Install dependencies
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-mamba: true
environment-file: conda-environments/docbuild.yml
python-version: "3.10"
activate-environment: docbuild
auto-activate-base: false
auto-update-conda: false
- name: Install activitysim
run: |
python -m pip install .
- name: Conda checkup
run: |
conda info -a
conda list
echo ${{ github.repository }}
echo ${{ github.ref_name }}
- name: localize version switcher
run: |
python .github/workflows/localize-base-urls.py docs/_static/switcher.json
git update-index --assume-unchanged docs/_static/switcher.json
cat docs/_static/switcher.json
- name: Build the docs
run: |
cd docs
make clean
make html
- name: Push to GitHub Pages
uses: peaceiris/[email protected]
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
# Token is created automatically by Github Actions, no other config needed
publish_dir: ./docs/_build/html
destination_dir: develop