Merge pull request #6 from eFiniLan/2023.08.24

2023.08.24

Author: eFiniLan

.devcontainer/.gitignore

@@ -0,0 +1 @@
+.Xauthority

.devcontainer/Dockerfile

@@ -0,0 +1,6 @@
+FROM ghcr.io/commaai/openpilot-base:latest
+
+# remove gitconfig if exists, since its gonna be replaced by host one
+RUN rm -f /root/.gitconfig
+RUN apt update && apt install -y vim net-tools usbutils htop ripgrep
+RUN pip install ipython jupyter jupyterlab

.devcontainer/devcontainer.json

@@ -0,0 +1,23 @@
+{
+  "name": "openpilot devcontainer",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "postCreateCommand": "bash -c 'if [[ $DISPLAY == *xquartz* ]]; then echo \"export DISPLAY=host.docker.internal:0\" >> /root/.bashrc; fi'",
+  "postStartCommand": "git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | xargs -I{} git config --global --add safe.directory \"$PWD/{}\"",
+  "initializeCommand": ".devcontainer/setup_host.sh",
+  "privileged": true,
+  "containerEnv": {
+    "DISPLAY": "${localEnv:DISPLAY}",
+    "PYTHONPATH": "${containerWorkspaceFolder}",
+    "force_color_prompt": "1"
+  },
+  "runArgs": [
+    "--volume=/tmp/.X11-unix:/tmp/.X11-unix",
+    "--volume=${localWorkspaceFolder}/.devcontainer/.Xauthority:/root/.Xauthority",
+    "--volume=${localEnv:HOME}/.comma:/root/.comma",
+    "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
+    "--volume=/tmp/devcontainer_scons_cache:/tmp/scons_cache",
+    "--shm-size=1G"
+  ]
+}

.devcontainer/setup_host.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# setup links to Xauthority
+XAUTHORITY_LINK=".devcontainer/.Xauthority"
+rm -f $XAUTHORITY_LINK
+if [[ -z $XAUTHORITY ]]; then
+    echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..."
+    if ! [[ -f $HOME/.Xauthority ]]; then
+        echo "~/.XAuthority file does not exist. GUI tools may not work properly."
+        touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount
+    else
+        ln -sf $HOME/.Xauthority $XAUTHORITY_LINK
+    fi
+else
+    ln -sf $XAUTHORITY $XAUTHORITY_LINK
+fi

.dockerignore

@@ -16,6 +16,9 @@
 *.so
 *.a
 
+venv/
+.venv/
+
 notebooks
 phone
 massivemap

.gitattributes

@@ -1,2 +1,3 @@
 *.dlc filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
+selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text

.github/workflows/badges.yaml

@@ -7,7 +7,7 @@ on:
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
 
 jobs:
   badges:

.github/workflows/prebuilt.yaml

@@ -10,7 +10,7 @@ env:
 
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+      DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
 
 jobs:
   build_prebuilt:
@@ -34,7 +34,7 @@ jobs:
     - name: Build Docker image
       run: |
         eval "$BUILD"
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
+        DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
     - name: Push to container registry
       run: |
         $DOCKER_LOGIN

.github/workflows/selfdrive_tests.yaml

@@ -2,29 +2,31 @@ name: selfdrive
 
 on:
   push:
-    branches-ignore:
-      - 'testing-closet*'
+    branches:
+      - master
   pull_request:
 
 concurrency:
   group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:
+  PYTHONWARNINGS: error
+
   BASE_IMAGE: openpilot-base
   CL_BASE_IMAGE: openpilot-base-cl
   DOCKER_REGISTRY: ghcr.io/commaai
   AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
 
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+      DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
 
   BUILD_CL: |
-      DOCKER_BUILDKIT=1 docker build --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+      DOCKER_BUILDKIT=1 docker build --cache-to type=inline --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
+  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   UNIT_TEST: coverage run --append -m unittest discover
 
@@ -39,30 +41,33 @@ jobs:
       with:
         submodules: true
     - name: Build devel
+      timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
     - uses: ./.github/workflows/setup
     - name: Check submodules
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
+      timeout-minutes: 1
       run: release/check-submodules.sh
     - name: Build openpilot and run checks
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: |
         cd $STRIPPED_DIR
         ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py"
     - name: Run tests
+      timeout-minutes: 2
       run: |
         cd $STRIPPED_DIR
         ${{ env.RUN }} "release/check-dirty.sh && \
                         python -m unittest discover selfdrive/car"
     - name: pre-commit
+      timeout-minutes: 3
       run: |
         cd $GITHUB_WORKSPACE
         cp .pre-commit-config.yaml $STRIPPED_DIR
-        cp .pylintrc $STRIPPED_DIR
-        cp mypy.ini $STRIPPED_DIR
         cp pyproject.toml $STRIPPED_DIR
         cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "pre-commit run --all"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
 
   build_all:
     name: build all
@@ -72,14 +77,20 @@ jobs:
       with:
         submodules: true
     - uses: ./.github/workflows/setup
-      with:
-        save-cache: true
     - name: Build openpilot with all flags
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
       run: ${{ env.RUN }} "scons -j$(nproc) --extras && release/check-dirty.sh"
-    - name: Cleanup scons cache
+    - name: Cleanup scons cache and rebuild
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 2 || 30) }} # allow more time when we missed the scons cache
       run: |
           ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
                           scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v3
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: ~/scons_cache
+        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
 
   build_mac:
     name: build macos
@@ -103,9 +114,11 @@ jobs:
         restore-keys: macos_scons-
     - name: Cache dependencies
       id: dependency-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: |
+          .env
+          .venv
           ~/github_brew_cache_entries.txt
           ~/.pyenv
           ~/Library/Caches/pypoetry
@@ -116,7 +129,7 @@ jobs:
           /opt/homebrew/opt
           /opt/homebrew/Caskroom/gcc-arm-*
           /Applications/ArmGNUToolchain/*/*/*
-        key: macos_deps-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }}
+        key: macos_deps-${{ hashFiles('tools/mac_setup.sh', 'tools/install_python_dependencies.sh', 'poetry.lock') }}
         restore-keys: macos_deps-
     - name: Brew link restored dependencies
       run: |
@@ -132,20 +145,22 @@ jobs:
     - name: Install dependencies
       if: steps.dependency-cache.outputs.cache-hit != 'true'
       run: ./tools/mac_setup.sh
+      env:
+        # package install has DeprecationWarnings
+        PYTHONWARNINGS: default
     - name: Build openpilot
       run: |
-        source tools/openpilot_env.sh
+        eval "$(pyenv init --path)"
         poetry run scons -j$(nproc)
     - name: Run tests
       run: |
-        source tools/openpilot_env.sh
-        export PYTHONPATH=$PWD
+        eval "$(pyenv init --path)"
         poetry run tools/plotjuggler/test_plotjuggler.py
     - name: Pre Cache - Cleanup scons cache
       if: github.ref == 'refs/heads/master'
       run: |
-        source tools/openpilot_env.sh
         rm -rf /tmp/scons_cache/*
+        eval "$(pyenv init --path)"
         poetry run scons -j$(nproc) --cache-populate
     - name: Save scons cache
       id: scons-save-cache
@@ -158,8 +173,9 @@ jobs:
       if: steps.dependency-cache.outputs.cache-hit != 'true'
       run: |
         new_cellar=$(brew list --formula -1)
+        exceptions="zstd lz4 xz" # caching step needs zstd
         comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
-          if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+          if [[ " $exceptions " != *" $pkg "* ]]; then
             rm -rf "$(brew --cellar)/$pkg"
           fi
         done
@@ -203,7 +219,8 @@ jobs:
     - name: Build Docker image
       run: eval "$BUILD"
     - name: pre-commit
-      run: ${{ env.RUN }} "pre-commit run --all"
+      timeout-minutes: 4
+      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
 
   valgrind:
     name: valgrind
@@ -216,6 +233,7 @@ jobs:
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run valgrind
+      timeout-minutes: 1
       run: |
         ${{ env.RUN }} "python selfdrive/test/test_valgrind_replay.py"
     - name: Print logs
@@ -231,8 +249,10 @@ jobs:
         submodules: true
     - uses: ./.github/workflows/setup
     - name: Build openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run unit tests
+      timeout-minutes: 40
       run: |
         ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
                         $UNIT_TEST common && \
@@ -243,6 +263,7 @@ jobs:
                         $UNIT_TEST system/loggerd && \
                         $UNIT_TEST selfdrive/car && \
                         $UNIT_TEST selfdrive/locationd && \
+                        $UNIT_TEST selfdrive/test/longitudinal_maneuvers && \
                         $UNIT_TEST system/tests && \
                         $UNIT_TEST system/ubloxd && \
                         selfdrive/locationd/test/_test_locationd_lib.py && \
@@ -255,6 +276,7 @@ jobs:
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py && \
                         ./common/tests/test_util && \
+                        ./common/tests/test_ratekeeper && \
                         ./common/tests/test_swaglog && \
                         ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
                         ./system/loggerd/tests/test_logger &&\
@@ -279,12 +301,13 @@ jobs:
       id: dependency-cache
       uses: actions/cache@v3
       with:
-        path: /tmp/comma_download_cache
+        path: ~/comma_download_cache
         key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run replay
+      timeout-minutes: 30
       run: |
         ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                         coverage xml"
@@ -321,43 +344,22 @@ jobs:
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
+    # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
     - name: Run model replay with ONNX
+      timeout-minutes: 2
       run: |
-        ${{ env.RUN_CL }} "ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
+        ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \
+                           ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
                            coverage xml"
     - name: Run unit tests
+      timeout-minutes: 4
       run: |
-        ${{ env.RUN_CL }} "$UNIT_TEST selfdrive/modeld && \
+        ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \
+                           $UNIT_TEST selfdrive/modeld && \
                            coverage xml"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
-  test_longitudinal:
-    name: longitudinal
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Test longitudinal
-      run: |
-        ${{ env.RUN }} "mkdir -p selfdrive/test/out && \
-                        cd selfdrive/test/longitudinal_maneuvers && \
-                        coverage run ./test_longitudinal.py && \
-                        coverage xml"
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v3
-    - uses: actions/upload-artifact@v2
-      if: always()
-      continue-on-error: true
-      with:
-        name: longitudinal
-        path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/
-
   test_cars:
     name: cars
     runs-on: ubuntu-20.04
@@ -372,16 +374,16 @@ jobs:
     - uses: ./.github/workflows/setup
     - name: Cache test routes
       id: dependency-cache
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+      uses: actions/cache@v3
       with:
-        path: /tmp/comma_download_cache
+        path: ~/comma_download_cache
         key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Test car models
+      timeout-minutes: 25
       run: |
-        ${{ env.RUN }} "coverage run -m pytest selfdrive/car/tests/test_models.py && \
-                        coverage xml && \
+        ${{ env.RUN }} "pytest --cov --cov-report=xml -n auto --dist=loadscope selfdrive/car/tests/test_models.py && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
         NUM_JOBS: 5
@@ -410,7 +412,7 @@ jobs:
         id: save_diff
         run: |
           ${{ env.RUN }} "scons -j$(nproc)"
-          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info") || true
+          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info")
           output="${output//$'\n'/'%0A'}"
           echo "::set-output name=diff::$output"
       - name: Find comment