Skip to content
CI

v3.0.0 - Continuous recordings #648

Sign in to view logs

v3.0.0 - Continuous recordings

v3.0.0 - Continuous recordings #648

Workflow file for this run

.github/workflows/ci.yaml at 218f063
name: CI
on:
push:
branches:
- dev
- master
pull_request: ~
env:
CACHE_VERSION: 1
DEFAULT_PYTHON: "3.10"
PRE_COMMIT_CACHE: ~/.cache/pre-commit
DOCKER_BUILDKIT: 1
COMPOSE_DOCKER_CLI_BUILD: 1
BUILDKIT_PROGRESS: plain
jobs:
prepare-python-venv:
name: Prepare Python venv
runs-on: ubuntu-22.04
outputs:
python-venv-cache-key: ${{ steps.python-venv-cache-key.outputs.key }}
steps:
# Create Python virtual env used for linters
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/[email protected]
with:
python-version: ${{ env.DEFAULT_PYTHON }}
# Create a virtual environment and cache it
- name: Generate Python virtual environment restore key
id: python-venv-cache-key
env:
cache-name: cache-python-venv
run: >-
echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
- name: Restore Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ steps.python-venv-cache-key.outputs.key }}
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends libgirepository1.0-dev
python3 -m venv --system-site-packages venv
- name: Install requirements into Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
uses: ./.github/templates/run_in_venv
with:
command: |
pip3 install -r requirements.txt -r requirements_test.txt -r requirements_ci.txt && \
pre-commit install
prepare-pre-commit:
name: Prepare pre-commit
runs-on: ubuntu-22.04
needs: prepare-python-venv
outputs:
pre-commit-cache-key: ${{ steps.pre-commit-cache-key.outputs.key }}
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Generate pre-commit restore key
id: pre-commit-cache-key
env:
cache-name: cache-pre-commit
run: >-
echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
# Install pre-commit hooks into the cached virtual environment
- name: Restore pre-commit hooks
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ steps.pre-commit-cache-key.outputs.key }}
- name: Create pre-commit hooks
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit install-hooks
run-isort:
name: Run isort
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run isort
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual isort --all-files --show-diff-on-failure
run-black:
name: Run black
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run black
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual black --all-files --show-diff-on-failure
run-codespell:
name: Run codespell
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run codespell
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual codespell --all-files --show-diff-on-failure
run-mypy:
name: Run mypy
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run mypy
uses: ./.github/templates/run_in_venv
with:
command: |
mypy --install-types --non-interactive viseron
pre-commit run --hook-stage manual mypy --all-files --show-diff-on-failure
run-pylint:
name: Run pylint
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends python3-gi python3-gst-1.0
- name: Run pylint
uses: ./.github/templates/run_in_venv
with:
command: |
pylint --version
pylint viseron
run-flake8:
name: Run flake8
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run flake8
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual flake8 --all-files --show-diff-on-failure
run-pyupgrade:
name: Run pyupgrade
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run pyupgrade
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual pyupgrade --all-files --show-diff-on-failure
run-pytest:
name: Run pytest
runs-on: ubuntu-22.04
needs: prepare-python-venv
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
with:
fetch-depth: 2
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Pull current Docker dev tag
run: |
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env pull amd64-viseron
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env pull amd64-wheels
- name: Re-build wheels
run: |
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-wheels
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-viseron
- name: Build pytest Docker image
run: |
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-viseron-tests
- name: Run pytest
run: |
if docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env up amd64-viseron-tests; then
exit 0
else
exit 1
fi
- name: Copy .coverage to host
run: |
docker cp amd64-viseron-tests:/src/coverage.xml coverage.xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
run-generated-docs:
name: Check generated docs
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/[email protected]
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/[email protected]
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Install libedgetpu1 for edgetpu docs
run: |
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
sudo add-apt-repository "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main"
sudo apt-get update
sudo apt-get install -y --no-install-recommends libedgetpu1-std python3-gi python3-gst-1.0
- name: Run script to check generated docs
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual generate_docs --all-files --show-diff-on-failure
pre-commit run --hook-stage manual check_config_json --all-files --show-diff-on-failure