From 35911067370ca0130608f1fec3eafaab2a4ce8e6 Mon Sep 17 00:00:00 2001 From: Yukinari Hisaki <42021302+yhisaki@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:17:28 +0900 Subject: [PATCH 1/6] feat(.clang-tidy): add ExtraArgs (#5439) Signed-off-by: Y.Hisaki --- .clang-tidy | 3 +++ .clang-tidy-ci | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.clang-tidy b/.clang-tidy index 06c54854b5d..f5eccdf3cc7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -202,6 +202,9 @@ WarningsAsErrors: " HeaderFilterRegex: ^(?!\/usr)(?!\/opt) +ExtraArgs: + - -std=c++17 + FormatStyle: none CheckOptions: diff --git a/.clang-tidy-ci b/.clang-tidy-ci index 04a9f8bacce..c6dfcdb37c7 100644 --- a/.clang-tidy-ci +++ b/.clang-tidy-ci @@ -225,6 +225,9 @@ WarningsAsErrors: " HeaderFilterRegex: ^(?!\/usr)(?!\/opt) +ExtraArgs: + - -std=c++17 + CheckOptions: - key: bugprone-argument-comment.CommentBoolLiterals value: "0" From d1cae7e0a35bdead9619a859738be17d1387f148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Fatih=20C=C4=B1r=C4=B1t?= Date: Mon, 18 Nov 2024 12:01:17 +0300 Subject: [PATCH 2/6] docs(ansible): update role readme files (#5443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: M. Fatih Cırıt --- ansible/roles/cuda/README.md | 34 +++++++++++++++++++++++-------- ansible/roles/dev_tools/README.md | 25 ++++++++--------------- ansible/roles/tensorrt/README.md | 22 +++++++------------- 3 files changed, 42 insertions(+), 39 deletions(-) diff --git a/ansible/roles/cuda/README.md b/ansible/roles/cuda/README.md index 57d638d1187..d669c573e91 100644 --- a/ansible/roles/cuda/README.md +++ b/ansible/roles/cuda/README.md @@ -1,6 +1,6 @@ # cuda -This role installs [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) following [this page](https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=22.04&target_type=deb_network) and [this page](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#post-installation-actions). +This role installs [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) following [this page](https://developer.nvidia.com/cuda-12-3-0-download-archive?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=22.04&target_type=deb_network) and [this page](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#post-installation-actions). This role also registers Vulkan, OpenGL, and OpenCL GPU vendors for future use. @@ -13,22 +13,36 @@ This role also registers Vulkan, OpenGL, and OpenCL GPU vendors for future use. ## Manual Installation -Follow these instructions to download and install the CUDA Toolkit and the corresponding NVIDIA Driver for Ubuntu 20.04. +### Version compatibility -For Universe, the `cuda_version` version can also be found in: -[../../playbooks/universe.yaml](../../playbooks/universe.yaml) +Autoware currently uses CUDA `12.3` and from this [CUDA Application Compatibility Support Matrix](https://docs.nvidia.com/deploy/cuda-compatibility/#use-the-right-compat-package) that only the NVIDIA driver version `545` is compatible with this version of CUDA. + +#### 🛠️ For Advanced Users + +⚠️ **Proceed with caution**: Avoid removing essential system components. + +To prevent conflicts during NVIDIA installation, we recommend completely removing old NVIDIA Drivers and CUDA by following this guide: [How can I uninstall a NVIDIA driver completely?](https://askubuntu.com/a/206289/761440). + +- **Important**: If you remove the previous NVIDIA drivers, ensure you install the recommended versions listed below **before restarting your system**. + +Once the drivers are installed correctly, you may safely restart your system. + +### CUDA Toolkit and Driver + +Follow these instructions to download and install the CUDA Toolkit. + +From: ```bash wget -O /tmp/amd64.env https://raw.githubusercontent.com/autowarefoundation/autoware/main/amd64.env && source /tmp/amd64.env -# From: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#network-repo-installation-for-ubuntu - os=ubuntu2204 wget https://developer.download.nvidia.com/compute/cuda/repos/$os/$(uname -m)/cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo apt-get update cuda_version_dashed=$(eval sed -e "s/[.]/-/g" <<< "${cuda_version}") -sudo apt-get -y install cuda-${cuda_version_dashed} +sudo apt-get -y install cuda-toolkit-${cuda_version_dashed} +sudo apt-get install -y cuda-drivers-545 ``` Perform the post installation actions: @@ -37,9 +51,13 @@ Perform the post installation actions: # Taken from: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#post-installation-actions echo 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' >> ~/.bashrc echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' >> ~/.bashrc +``` + +### GPU Vendors -# Register Vulkan, OpenGL, and OpenCL GPU vendors +Register Vulkan, OpenGL, and OpenCL GPU vendors following the instructions below. +```bash # Create Vulkan directory sudo mkdir -p /etc/vulkan/icd.d sudo chmod 0755 /etc/vulkan/icd.d diff --git a/ansible/roles/dev_tools/README.md b/ansible/roles/dev_tools/README.md index 145c70f020b..89d51fbdc3b 100644 --- a/ansible/roles/dev_tools/README.md +++ b/ansible/roles/dev_tools/README.md @@ -12,34 +12,27 @@ This role installs optional development tools for Autoware. ## Inputs -| Name | Required | Description | -| ------------- | -------- | --------------------------------------- | -| clang-version | true | The version of clang-format to install. | -| ros-distro | true | The ROS distribution. | +| Name | Required | Description | +| ------------------------------- | -------- | --------------------------------------- | +| pre_commit_clang_format_version | true | The version of clang-format to install. | +| rosdistro | true | The ROS distribution. | ## Manual Installation ```bash -#!/bin/bash +# For the environment variables +wget -O /tmp/amd64.env https://raw.githubusercontent.com/autowarefoundation/autoware/main/amd64.env && source /tmp/amd64.env -# Update package lists sudo apt-get update -# Install Git LFS +sudo apt install python3-pip +sudo apt-get install -y golang +sudo apt-get install -y ros-${rosdistro}-plotjuggler-ros sudo apt-get install -y git-lfs # Setup Git LFS git lfs install -# Install pre-commit using pip3 pip3 install pre-commit - -# Install a specific version of clang-format using pip3 pip3 install clang-format==${pre_commit_clang_format_version} - -# Install Go -sudo apt-get install -y golang - -# Install PlotJuggler -sudo apt-get install -y ros-${ROS_DISTRO}-plotjuggler-ros ``` diff --git a/ansible/roles/tensorrt/README.md b/ansible/roles/tensorrt/README.md index 1df87cad89e..2a6c7d39232 100644 --- a/ansible/roles/tensorrt/README.md +++ b/ansible/roles/tensorrt/README.md @@ -1,6 +1,6 @@ # tensorrt -This role installs TensorRT and cuDNN following [this page](https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#installing). +This role installs TensorRT and cuDNN following [the official NVIDIA TensorRT Installation Guide](https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#installing). ## Inputs @@ -11,29 +11,16 @@ This role installs TensorRT and cuDNN following [this page](https://docs.nvidia. ## Manual Installation -For Universe, the `cudnn_version` and `tensorrt_version` variables should be copied from -[amd64.env](../../../amd64.env) or [arm64.env](../../../arm64.env) depending on the architecture used. - ```bash +# For the environment variables wget -O /tmp/amd64.env https://raw.githubusercontent.com/autowarefoundation/autoware/main/amd64.env && source /tmp/amd64.env -# Can also be found at: https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#installing - sudo apt-get install -y \ libcudnn8=${cudnn_version} \ libnvinfer8=${tensorrt_version} \ libnvinfer-plugin8=${tensorrt_version} \ libnvparsers8=${tensorrt_version} \ libnvonnxparsers8=${tensorrt_version} \ - -sudo apt-mark hold \ -libcudnn8 \ -libnvinfer8 \ -libnvinfer-plugin8 \ -libnvparsers8 \ -libnvonnxparsers8 - -sudo apt-get install -y \ libcudnn8-dev=${cudnn_version} \ libnvinfer-dev=${tensorrt_version} \ libnvinfer-plugin-dev=${tensorrt_version} \ @@ -43,6 +30,11 @@ libnvparsers-dev=${tensorrt_version} \ libnvonnxparsers-dev=${tensorrt_version} sudo apt-mark hold \ +libcudnn8 \ +libnvinfer8 \ +libnvinfer-plugin8 \ +libnvparsers8 \ +libnvonnxparsers8 \ libcudnn8-dev \ libnvinfer-dev \ libnvinfer-plugin-dev \ From ef7a992c52f54269a8b7773f6d26c218ec6081dd Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Mon, 18 Nov 2024 18:14:38 +0900 Subject: [PATCH 3/6] refactor(docker): replace the multiple `apt-get autoremove` commands with `cleanup_apt.sh`. (#5447) --- docker/Dockerfile | 56 +++++++++++++++++++++-------------- docker/scripts/cleanup_apt.sh | 11 +++++++ 2 files changed, 45 insertions(+), 22 deletions(-) create mode 100755 docker/scripts/cleanup_apt.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 5534205363e..1bf5b246c4b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,6 +5,15 @@ FROM $BASE_IMAGE AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO +# Copy files +COPY setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/ +COPY ansible/ /autoware/ansible/ +COPY docker/scripts/cleanup_apt.sh /autoware/cleanup_apt.sh +RUN chmod +x /autoware/cleanup_apt.sh +COPY docker/scripts/cleanup_system.sh /autoware/cleanup_system.sh +RUN chmod +x /autoware/cleanup_system.sh +WORKDIR /autoware + # Install apt packages and add GitHub to known hosts for private repositories RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache @@ -12,23 +21,16 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ gosu \ ssh \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ + && /autoware/cleanup_apt.sh \ && mkdir -p ~/.ssh \ && ssh-keyscan github.com >> ~/.ssh/known_hosts -# Copy files -COPY setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/ -COPY ansible/ /autoware/ansible/ -COPY docker/scripts/cleanup_system.sh /autoware/cleanup_system.sh -RUN chmod +x /autoware/cleanup_system.sh -WORKDIR /autoware - # Set up base environment RUN --mount=type=ssh \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ ./setup-dev-env.sh -y --module base --no-nvidia --no-cuda-drivers --runtime openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache \ + && /autoware/cleanup_apt.sh \ && echo "source /opt/ros/${ROS_DISTRO}/setup.bash" > /etc/bash.bashrc # Create entrypoint @@ -45,7 +47,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN --mount=type=ssh \ ./setup-dev-env.sh -y --module base --download-artifacts --no-cuda-drivers --runtime openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache + && /autoware/cleanup_apt.sh true # hadolint ignore=DL3006 FROM $BASE_IMAGE AS rosdep-depend @@ -54,6 +56,8 @@ ARG ROS_DISTRO COPY setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/ COPY ansible/ /autoware/ansible/ +COPY docker/scripts/cleanup_apt.sh /autoware/cleanup_apt.sh +RUN chmod +x /autoware/cleanup_apt.sh COPY docker/scripts/resolve_rosdep_keys.sh /autoware/resolve_rosdep_keys.sh RUN chmod +x /autoware/resolve_rosdep_keys.sh WORKDIR /autoware @@ -64,7 +68,7 @@ RUN --mount=type=ssh \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ ./setup-dev-env.sh -y rosdep \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache + && /autoware/cleanup_apt.sh # Generate install package lists COPY src/core /autoware/src/core @@ -169,7 +173,7 @@ RUN --mount=type=ssh \ ./setup-dev-env.sh -y --module all --no-nvidia --no-cuda-drivers openadkit \ && ./setup-dev-env.sh -y --module dev-tools openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache + && /autoware/cleanup_apt.sh # Install rosdep dependencies COPY --from=rosdep-depend /rosdep-core-depend-packages.txt /tmp/rosdep-core-depend-packages.txt @@ -177,7 +181,7 @@ COPY --from=rosdep-depend /rosdep-core-depend-packages.txt /tmp/rosdep-core-depe RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-core-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache + && /autoware/cleanup_apt.sh RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-depend,source=/autoware/src/core,target=/autoware/src/core \ @@ -196,9 +200,10 @@ ENV CCACHE_DIR="/root/.ccache" COPY --from=rosdep-depend /rosdep-universe-common-depend-packages.txt /tmp/rosdep-universe-common-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-common-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ @@ -219,7 +224,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN --mount=type=ssh \ ./setup-dev-env.sh -y --module all --no-cuda-drivers --runtime openadkit \ && pip uninstall -y ansible ansible-core \ - && apt-get autoremove -y && rm -rf "$HOME"/.cache + && /autoware/cleanup_apt.sh true ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] @@ -233,9 +238,10 @@ ENV CCACHE_DIR="/root/.ccache" COPY --from=rosdep-universe-sensing-perception-depend /rosdep-universe-sensing-perception-depend-packages.txt /tmp/rosdep-universe-sensing-perception-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-sensing-perception-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ @@ -257,9 +263,10 @@ ENV CCACHE_DIR="/root/.ccache" COPY --from=rosdep-universe-sensing-perception-depend /rosdep-universe-sensing-perception-depend-packages.txt /tmp/rosdep-universe-sensing-perception-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-sensing-perception-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ @@ -293,9 +300,10 @@ ENV CCACHE_DIR="/root/.ccache" COPY --from=rosdep-universe-localization-mapping-depend /rosdep-universe-localization-mapping-depend-packages.txt /tmp/rosdep-universe-localization-mapping-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-localization-mapping-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ @@ -314,9 +322,10 @@ ENV CCACHE_DIR="/root/.ccache" COPY --from=rosdep-universe-planning-control-depend /rosdep-universe-planning-control-depend-packages.txt /tmp/rosdep-universe-planning-control-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-planning-control-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ @@ -343,9 +352,10 @@ ENV CCACHE_DIR="/root/.ccache" COPY --from=rosdep-universe-vehicle-system-depend /rosdep-universe-vehicle-system-depend-packages.txt /tmp/rosdep-universe-vehicle-system-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-vehicle-system-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ @@ -369,9 +379,10 @@ ENV CCACHE_DIR="/root/.ccache" COPY --from=rosdep-universe-depend /rosdep-universe-depend-packages.txt /tmp/rosdep-universe-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh COPY --from=universe-sensing-perception-devel /opt/autoware /opt/autoware COPY --from=universe-localization-mapping-devel /opt/autoware /opt/autoware @@ -402,9 +413,10 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] COPY --from=rosdep-universe-depend /rosdep-universe-depend-packages.txt /tmp/rosdep-universe-depend-packages.txt # hadolint ignore=SC2002 RUN --mount=type=ssh \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update \ && cat /tmp/rosdep-universe-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache + && /autoware/cleanup_apt.sh COPY --from=universe-sensing-perception-devel-cuda /opt/autoware /opt/autoware COPY --from=universe-devel /opt/autoware /opt/autoware diff --git a/docker/scripts/cleanup_apt.sh b/docker/scripts/cleanup_apt.sh new file mode 100755 index 00000000000..883df9ba8af --- /dev/null +++ b/docker/scripts/cleanup_apt.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +function cleanup() { + local remove_var_lib_apt_lists=false + apt-get autoremove -y && rm -rf "$HOME"/.cache + if "$remove_var_lib_apt_lists"; then + rm -rf /var/lib/apt/lists/* + fi +} + +cleanup "$@" From 0dd08838d0ebc154bf698ee1b8d157a456eb044f Mon Sep 17 00:00:00 2001 From: oguzkaganozt Date: Mon, 18 Nov 2024 16:49:04 +0300 Subject: [PATCH 4/6] fix(docker): add autoware prefix to map_loader (#5451) add autoware prefix to map_loader Signed-off-by: Oguz Ozturk --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1bf5b246c4b..f6338b4a171 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -115,7 +115,7 @@ ARG ROS_DISTRO COPY src/universe/autoware.universe/control /autoware/src/universe/autoware.universe/control COPY src/universe/autoware.universe/planning /autoware/src/universe/autoware.universe/planning # TODO(youtalk): Remove COPYs when https://github.com/autowarefoundation/autoware.universe/issues/8805 is resolved -COPY src/universe/autoware.universe/map/map_loader /autoware/src/universe/autoware.universe/map/map_loader +COPY src/universe/autoware.universe/map/autoware_map_loader /autoware/src/universe/autoware.universe/map/autoware_map_loader COPY src/universe/autoware.universe/map/autoware_map_projection_loader /autoware/src/universe/autoware.universe/map/autoware_map_projection_loader COPY src/universe/autoware.universe/sensing/autoware_pcl_extensions /autoware/src/universe/autoware.universe/sensing/autoware_pcl_extensions COPY src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor /autoware/src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor @@ -332,7 +332,7 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/control,target=/autoware/src/universe/autoware.universe/control \ --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/planning,target=/autoware/src/universe/autoware.universe/planning \ # TODO(youtalk): Remove --mount options when https://github.com/autowarefoundation/autoware.universe/issues/8805 is resolved - --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/map/map_loader,target=/autoware/src/universe/autoware.universe/map/map_loader \ + --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/map/autoware_map_loader,target=/autoware/src/universe/autoware.universe/map/autoware_map_loader \ --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/map/autoware_map_projection_loader,target=/autoware/src/universe/autoware.universe/map/autoware_map_projection_loader \ --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/sensing/autoware_pcl_extensions,target=/autoware/src/universe/autoware.universe/sensing/autoware_pcl_extensions \ --mount=type=bind,from=rosdep-universe-planning-control-depend,source=/autoware/src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor,target=/autoware/src/universe/autoware.universe/sensing/autoware_pointcloud_preprocessor \ From 39f266a55269e8cfa1ad7d04d8495bded13da48a Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Tue, 19 Nov 2024 11:56:12 +0900 Subject: [PATCH 5/6] fix(docker): install CUDA developement libraries into `-devel-cuda` images (#5446) * remove --runtime option Signed-off-by: Yutaka Kondo * add cleanup_aptget Signed-off-by: Yutaka Kondo * chmod +x Signed-off-by: Yutaka Kondo * use script Signed-off-by: Yutaka Kondo * style(pre-commit): autofix * install to rosdep stage Signed-off-by: Yutaka Kondo * fix spell check Signed-off-by: Yutaka Kondo * MUST REVERT Signed-off-by: Yutaka Kondo * change order Signed-off-by: Yutaka Kondo * Revert "MUST REVERT" This reverts commit 3fb57f620876b2a7f8f00a797370d52eaa25d1dd. --------- Signed-off-by: Yutaka Kondo Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f6338b4a171..8e9c9532a98 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -222,7 +222,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Set up CUDA development environment # hadolint ignore=SC2002 RUN --mount=type=ssh \ - ./setup-dev-env.sh -y --module all --no-cuda-drivers --runtime openadkit \ + ./setup-dev-env.sh -y --module all --no-cuda-drivers openadkit \ && pip uninstall -y ansible ansible-core \ && /autoware/cleanup_apt.sh true @@ -268,6 +268,8 @@ RUN --mount=type=ssh \ && cat /tmp/rosdep-universe-sensing-perception-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ && /autoware/cleanup_apt.sh +COPY --from=universe-sensing-perception-devel /opt/autoware /opt/autoware + # hadolint ignore=SC1091 RUN --mount=type=cache,target=${CCACHE_DIR} \ # TODO(youtalk): Move CUDA related packages into a dedicated directory @@ -286,8 +288,6 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ && source /opt/autoware/setup.bash \ && /autoware/build_and_clean.sh ${CCACHE_DIR} /opt/autoware -COPY --from=universe-sensing-perception-devel /opt/autoware /opt/autoware - ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["/bin/bash"] From abb6b8d3061b5416f0b02ec41d57069aca289ef5 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Tue, 19 Nov 2024 21:05:57 +0900 Subject: [PATCH 6/6] feat: introduce `autoware-nightly.repos` (#5440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add autoware-nightly.repos Signed-off-by: Yutaka Kondo * add health-check-nightly Signed-off-by: Yutaka Kondo * fix inputs Signed-off-by: Yutaka Kondo * rename Signed-off-by: Yutaka Kondo * change chache condition Signed-off-by: Yutaka Kondo * update .repos Signed-off-by: Yutaka Kondo * vcs import with multiple repos files Signed-off-by: Yutaka Kondo * fix conditions Signed-off-by: Yutaka Kondo * refine Signed-off-by: Yutaka Kondo * style(pre-commit): autofix * remove comment Signed-off-by: Yutaka Kondo * wip Signed-off-by: Yutaka Kondo * Revert "wip" This reverts commit 65e89d1e32a1f591204d5004b1dddd87af99950f. * wip Signed-off-by: Yutaka Kondo * Update action.yaml Co-authored-by: M. Fatih Cırıt --------- Signed-off-by: Yutaka Kondo Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: M. Fatih Cırıt --- .github/actions/docker-build/action.yaml | 18 +++++-- .github/workflows/health-check-nightly.yaml | 58 +++++++++++++++++++++ autoware-nightly.repos | 37 +++++++++++++ 3 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/health-check-nightly.yaml create mode 100644 autoware-nightly.repos diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index 9a6a6e7e140..df338a836d0 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -8,6 +8,10 @@ inputs: cache-tag-suffix: description: Suffix of the target cache tag. required: true + additional-repos: + description: Additional target .repos file. + default: "" + required: false build-args: description: Additional build args. required: false @@ -31,9 +35,15 @@ runs: vcs import src < autoware.repos shell: bash + - name: Import additional repositories + if: ${{ inputs.additional-repos != '' }} + run: | + vcs import src < ${{ inputs.additional-repos }} + shell: bash + - name: Cache ccache uses: actions/cache@v4 - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' && inputs.additional-repos == '' }} id: cache-ccache with: path: | @@ -44,7 +54,7 @@ runs: - name: Cache apt-get uses: actions/cache@v4 - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' && inputs.additional-repos == '' }} id: cache-apt-get with: path: | @@ -55,7 +65,7 @@ runs: - name: Restore ccache uses: actions/cache/restore@v4 - if: ${{ github.ref != 'refs/heads/main' }} + if: ${{ github.ref != 'refs/heads/main' || inputs.additional-repos != '' }} with: path: | root-ccache @@ -65,7 +75,7 @@ runs: - name: Restore apt-get uses: actions/cache/restore@v4 - if: ${{ github.ref != 'refs/heads/main' }} + if: ${{ github.ref != 'refs/heads/main' || inputs.additional-repos != '' }} with: path: | var-cache-apt diff --git a/.github/workflows/health-check-nightly.yaml b/.github/workflows/health-check-nightly.yaml new file mode 100644 index 00000000000..f38836e5d1b --- /dev/null +++ b/.github/workflows/health-check-nightly.yaml @@ -0,0 +1,58 @@ +name: health-check-nightly + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + schedule: + - cron: 0 6 * * * + workflow_dispatch: + +jobs: + label-check: + uses: autowarefoundation/autoware-github-actions/.github/workflows/make-sure-label-is-present.yaml@v1 + with: + label: tag:run-health-check + + load-env: + needs: label-check + if: ${{ needs.label-check.outputs.result == 'true' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/load-env.yaml + + docker-build-nightly: + needs: load-env + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Free disk space + uses: ./.github/actions/free-disk-space + + - name: Build 'Autoware' + uses: ./.github/actions/docker-build + with: + platform: amd64 + cache-tag-suffix: nightly + additional-repos: autoware-nightly.repos + build-args: | + ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }} + BASE_IMAGE=${{ needs.load-env.outputs.base_image }} + LIB_DIR=x86_64 + + - name: Show disk space + if: always() + run: | + df -h diff --git a/autoware-nightly.repos b/autoware-nightly.repos new file mode 100644 index 00000000000..0b58999fd41 --- /dev/null +++ b/autoware-nightly.repos @@ -0,0 +1,37 @@ +repositories: + core/autoware.core: + type: git + url: https://github.com/autowarefoundation/autoware.core.git + version: main + universe/autoware.universe: + type: git + url: https://github.com/autowarefoundation/autoware.universe.git + version: main + launcher/autoware_launch: + type: git + url: https://github.com/autowarefoundation/autoware_launch.git + version: main + sensor_kit/sample_sensor_kit_launch: + type: git + url: https://github.com/autowarefoundation/sample_sensor_kit_launch.git + version: main + sensor_kit/external/awsim_sensor_kit_launch: # TODO: Integrate into sample_sensor_kit_launch + type: git + url: https://github.com/tier4/awsim_sensor_kit_launch.git + version: main + sensor_kit/awsim_labs_sensor_kit_launch: + type: git + url: https://github.com/autowarefoundation/awsim_labs_sensor_kit_launch.git + version: main + sensor_kit/single_lidar_sensor_kit_launch: + type: git + url: https://github.com/autowarefoundation/single_lidar_sensor_kit_launch.git + version: main + vehicle/sample_vehicle_launch: + type: git + url: https://github.com/autowarefoundation/sample_vehicle_launch.git + version: main + vehicle/awsim_labs_vehicle_launch: + type: git + url: https://github.com/autowarefoundation/awsim_labs_vehicle_launch.git + version: main