From 1a40fafe3857431ffc06a35b399440b6ee84835b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20B=C3=B6ther?= <2116466+MaxiBoether@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:07:41 +0100 Subject: [PATCH] Switch from `ubuntu-latest` to `ubuntu-22.04` in CI, update some dependencies (#644) --- .clang-tidy | 2 + .github/actions/mamba/action.yml | 4 +- .github/workflows/workflow.yaml | 210 +++++++++++++++++++++++++++---- CMakeLists.txt | 5 + cmake/dependencies.cmake | 8 +- cmake/storage_dependencies.cmake | 6 +- docker/Dependencies/Dockerfile | 2 +- modyn/storage/src/main.cpp | 14 +-- 8 files changed, 207 insertions(+), 44 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 275803f60..e911e0410 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -65,6 +65,8 @@ Checks: > -readability-magic-numbers, -readability-uppercase-literal-suffix, -misc-confusable-identifiers, + -portability-simd-intrinsics, + -clang-diagnostic-unknown-warning-option, ### Reasons for exclusions ## Generally not applicable # abseil we don't use the abseil library diff --git a/.github/actions/mamba/action.yml b/.github/actions/mamba/action.yml index 1d44895ed..8dfffda19 100644 --- a/.github/actions/mamba/action.yml +++ b/.github/actions/mamba/action.yml @@ -6,8 +6,8 @@ runs: with: python-version: "3.11" - - name: Install prerequisites for pystack - run: sudo apt-get install --no-install-recommends -qy libdw-dev libelf-dev + - name: Install prerequisites + run: sudo apt-get install --no-install-recommends -qy libdw-dev libelf-dev libsqlite3-dev shell: bash - name: Set cache date diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 4642d4bf7..6d080e92c 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -18,7 +18,7 @@ jobs: pre-commit-checks: name: Pre-commit Checks timeout-minutes: 30 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout branch uses: actions/checkout@v4 @@ -28,7 +28,7 @@ jobs: mypy-typechecking: name: Mypy Type Checks - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 20 defaults: run: @@ -52,7 +52,7 @@ jobs: pylint: timeout-minutes: 20 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check out code @@ -68,7 +68,7 @@ jobs: unittests: timeout-minutes: 25 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check out code @@ -90,7 +90,7 @@ jobs: format: timeout-minutes: 20 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -102,7 +102,7 @@ jobs: tidy: timeout-minutes: 60 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: CLANG_TIDY: clang-tidy-15 RUN_CLANG_TIDY: run-clang-tidy-15 @@ -113,7 +113,7 @@ jobs: - name: Install clang-tidy run: | sudo apt update - sudo apt install -y clang-tidy-15 + sudo apt install -y clang-tidy-15 libsqlite3-dev cmake --version - name: Configure CMake @@ -126,7 +126,7 @@ jobs: cpp_build_and_test: name: Build + Test (C++) - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 60 outputs: line-coverage: ${{steps.run_test_with_coverage.outputs.LINE_COVERAGE}} @@ -147,10 +147,6 @@ jobs: build-type: Tsan - compiler: { c: clang, cxx: clang++, version: 14 } build-type: Asan - exclude: - # gcc-12 causes a false-positive memory error in release (https://github.com/google/googletest/issues/4108) - - compiler: { c: gcc, cxx: g++, version: 12 } - build-type: Release env: CC: ${{matrix.compiler.c}}-${{matrix.compiler.version}} @@ -160,14 +156,14 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install ccache + - name: Install ccache and sqlite run: | sudo apt update - sudo apt install -y ccache + sudo apt install -y ccache libsqlite3-dev - name: Install clang version - if: ${{ matrix.compiler.version > 14 }} - uses: KyleMayes/install-llvm-action@v1 + if: ${{ matrix.compiler.c == 'clang' && matrix.compiler.version > 14 }} + uses: KyleMayes/install-llvm-action@v2.0.5 with: version: ${{ matrix.compiler.version }}.0 env: true @@ -227,7 +223,7 @@ jobs: cpp_coverage_main: name: C++ Test Coverage (gets coverage of main branch, currently not main branch because no C++ on main) - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 60 env: CC: clang-17 @@ -278,7 +274,7 @@ jobs: cpp_comment_on_pr: if: github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 name: Comment Coverage Results needs: [cpp_build_and_test, cpp_coverage_main] timeout-minutes: 5 @@ -303,7 +299,7 @@ jobs: # Checks whether the base container works correctly. dockerized-unittests: timeout-minutes: 180 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - pre-commit-checks - mypy-typechecking @@ -325,7 +321,7 @@ jobs: integrationtests-debug: timeout-minutes: 180 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - pre-commit-checks - mypy-typechecking @@ -337,12 +333,54 @@ jobs: - name: Check out code uses: actions/checkout@v4 + - name: Check disk space 1 + run: df . -h + + - name: Free disk space + run: | + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + sudo rm -rf \ + /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /usr/lib/jvm || true + echo "some directories deleted" + sudo apt install aptitude -y >/dev/null 2>&1 + sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ + esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ + google-cloud-sdk imagemagick \ + libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ + mercurial apt-transport-https mono-complete libmysqlclient \ + unixodbc-dev yarn chrpath libssl-dev libxft-dev \ + libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ + snmp pollinate libpq-dev postgresql-client powershell ruby-full \ + sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ + -y -f >/dev/null 2>&1 + sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 + sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 + sudo apt-get autoremove -y >/dev/null 2>&1 + sudo apt-get autoclean -y >/dev/null 2>&1 + echo "some packages purged" + + - name: Check disk space 2 + run: df . -h + + - name: Get more space + run: | + sudo rm -rf ${GITHUB_WORKSPACE}/.git + + - name: Check disk space 3 + run: df . -h + - name: Start docker compose and exit when tests run through run: bash scripts/run_integrationtests.sh Debug integrationtests-asan: timeout-minutes: 180 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - pre-commit-checks - mypy-typechecking @@ -353,12 +391,54 @@ jobs: - name: Check out code uses: actions/checkout@v4 + - name: Check disk space 1 + run: df . -h + + - name: Free disk space + run: | + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + sudo rm -rf \ + /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /usr/lib/jvm || true + echo "some directories deleted" + sudo apt install aptitude -y >/dev/null 2>&1 + sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ + esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ + google-cloud-sdk imagemagick \ + libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ + mercurial apt-transport-https mono-complete libmysqlclient \ + unixodbc-dev yarn chrpath libssl-dev libxft-dev \ + libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ + snmp pollinate libpq-dev postgresql-client powershell ruby-full \ + sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ + -y -f >/dev/null 2>&1 + sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 + sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 + sudo apt-get autoremove -y >/dev/null 2>&1 + sudo apt-get autoclean -y >/dev/null 2>&1 + echo "some packages purged" + + - name: Check disk space 2 + run: df . -h + + - name: Get more space + run: | + sudo rm -rf ${GITHUB_WORKSPACE}/.git + + - name: Check disk space 3 + run: df . -h + - name: Start docker compose and exit when tests run through run: bash scripts/run_integrationtests.sh Asan integrationtests-tsan: timeout-minutes: 180 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - pre-commit-checks - mypy-typechecking @@ -370,12 +450,54 @@ jobs: - name: Check out code uses: actions/checkout@v4 + - name: Check disk space 1 + run: df . -h + + - name: Free disk space + run: | + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + sudo rm -rf \ + /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /usr/lib/jvm || true + echo "some directories deleted" + sudo apt install aptitude -y >/dev/null 2>&1 + sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ + esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ + google-cloud-sdk imagemagick \ + libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ + mercurial apt-transport-https mono-complete libmysqlclient \ + unixodbc-dev yarn chrpath libssl-dev libxft-dev \ + libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ + snmp pollinate libpq-dev postgresql-client powershell ruby-full \ + sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ + -y -f >/dev/null 2>&1 + sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 + sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 + sudo apt-get autoremove -y >/dev/null 2>&1 + sudo apt-get autoclean -y >/dev/null 2>&1 + echo "some packages purged" + + - name: Check disk space 2 + run: df . -h + + - name: Get more space + run: | + sudo rm -rf ${GITHUB_WORKSPACE}/.git + + - name: Check disk space 3 + run: df . -h + - name: Start docker compose and exit when tests run through run: bash scripts/run_integrationtests.sh Tsan integrationtests-release: timeout-minutes: 180 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: - pre-commit-checks - mypy-typechecking @@ -387,5 +509,47 @@ jobs: - name: Check out code uses: actions/checkout@v4 + - name: Check disk space 1 + run: df . -h + + - name: Free disk space + run: | + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + sudo rm -rf \ + /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /usr/lib/jvm || true + echo "some directories deleted" + sudo apt install aptitude -y >/dev/null 2>&1 + sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ + esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ + google-cloud-sdk imagemagick \ + libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ + mercurial apt-transport-https mono-complete libmysqlclient \ + unixodbc-dev yarn chrpath libssl-dev libxft-dev \ + libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ + snmp pollinate libpq-dev postgresql-client powershell ruby-full \ + sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ + -y -f >/dev/null 2>&1 + sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 + sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true + sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 + sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 + sudo apt-get autoremove -y >/dev/null 2>&1 + sudo apt-get autoclean -y >/dev/null 2>&1 + echo "some packages purged" + + - name: Check disk space 2 + run: df . -h + + - name: Get more space + run: | + sudo rm -rf ${GITHUB_WORKSPACE}/.git + + - name: Check disk space 3 + run: df . -h + - name: Start docker compose and exit when tests run through run: bash scripts/run_integrationtests.sh Release diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fa3260a4..2cb299553 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,8 +55,13 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # Without this, we get compilation errors for the chrono header. We need to set this globally set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -std=c++20") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi") + +elseif (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + list(APPEND MODYN_COMPILE_OPTIONS "-Wno-missing-requires" "-Wno-unknown-warning-option") # causes issues since gcc 12 and grpc 1.69 endif() + + if (${MODYN_TEST_COVERAGE}) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Running with coverage flags") diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index c70558ff9..6cca7204b 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -8,7 +8,7 @@ message(STATUS "Making fmt available.") FetchContent_Declare( fmt GIT_REPOSITORY https://github.com/fmtlib/fmt.git - GIT_TAG 10.1.1 + GIT_TAG 10.2.1 ) FetchContent_MakeAvailable(fmt) @@ -18,7 +18,7 @@ set(SPDLOG_FMT_EXTERNAL ON) # Otherwise, we run into linking errors since the fm FetchContent_Declare( spdlog GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.12.0 + GIT_TAG v1.15.0 ) FetchContent_MakeAvailable(spdlog) @@ -27,7 +27,7 @@ message(STATUS "Making argparse available.") FetchContent_Declare( argparse GIT_REPOSITORY https://github.com/p-ranav/argparse.git - GIT_TAG v2.9 + GIT_TAG v3.1 ) FetchContent_MakeAvailable(argparse) @@ -37,7 +37,7 @@ message(STATUS "Making googletest available.") FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.14.0 + GIT_TAG v1.15.2 ) FetchContent_MakeAvailable(googletest) diff --git a/cmake/storage_dependencies.cmake b/cmake/storage_dependencies.cmake index 772df5ea6..7dabd3c2e 100644 --- a/cmake/storage_dependencies.cmake +++ b/cmake/storage_dependencies.cmake @@ -17,7 +17,7 @@ message(STATUS "Making rapidcsv available.") FetchContent_Declare( rapidcsv GIT_REPOSITORY https://github.com/d99kris/rapidcsv.git - GIT_TAG v8.80 + GIT_TAG v8.84 ) FetchContent_MakeAvailable(rapidcsv) @@ -102,7 +102,7 @@ if((NOT MODYN_TRY_LOCAL_GRPC) OR (NOT gRPC_FOUND)) FetchContent_Declare( gRPC GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG v1.59.2 # When updating this, make sure to also update the modynbase dockerfile + GIT_TAG v1.64.3 # When updating this, make sure to also update the modynbase dockerfile GIT_SHALLOW TRUE ) set(gRPC_BUILD_TESTS OFF CACHE BOOL "" FORCE) @@ -115,7 +115,7 @@ if((NOT MODYN_TRY_LOCAL_GRPC) OR (NOT gRPC_FOUND)) endif() file(DOWNLOAD -https://raw.githubusercontent.com/protocolbuffers/protobuf/v23.1/cmake/protobuf-generate.cmake +https://raw.githubusercontent.com/protocolbuffers/protobuf/v29.2/cmake/protobuf-generate.cmake ${CMAKE_CURRENT_BINARY_DIR}/protobuf-generate.cmake) include(${CMAKE_CURRENT_BINARY_DIR}/protobuf-generate.cmake) diff --git a/docker/Dependencies/Dockerfile b/docker/Dependencies/Dockerfile index 9964f07a4..f6d6d5b65 100644 --- a/docker/Dependencies/Dockerfile +++ b/docker/Dependencies/Dockerfile @@ -49,7 +49,7 @@ RUN echo "Used buildtype is ${MODYN_BUILDTYPE}" >> /src/.modyn_buildtype RUN echo "Used dependency buildtype is ${MODYN_DEP_BUILDTYPE}" >> /src/.modyn_dep_buildtype # Install gRPC systemwide. When updating the version, make sure to also update the storage_dependencies.cmake file -RUN git clone --recurse-submodules -b v1.59.2 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ +RUN git clone --recurse-submodules -b v1.64.3 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ cd grpc && mkdir -p cmake/build && cd cmake/build && \ cmake -DgRPC_PROTOBUF_PROVIDER=module -DABSL_ENABLE_INSTALL=On -DgRPC_BUILD_CSHARP_EXT=Off -DABSL_BUILD_TESTING=Off -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=${MODYN_DEP_BUILDTYPE} ../.. && \ make -j8 && make install && cd ../../ diff --git a/modyn/storage/src/main.cpp b/modyn/storage/src/main.cpp index 65a520e6b..b5418cea4 100644 --- a/modyn/storage/src/main.cpp +++ b/modyn/storage/src/main.cpp @@ -11,23 +11,15 @@ using namespace modyn::storage; void setup_logger() { spdlog::set_pattern("[%Y-%m-%d:%H:%M:%S] [%s:%#] [%l] [p%P:t%t] %v"); } -argparse::ArgumentParser setup_argparser() { - argparse::ArgumentParser parser("Modyn Storage"); - - parser.add_argument("config").help("Modyn infrastructure configuration file"); - - return parser; -} - int main(int argc, char* argv[]) { /* Entrypoint for the storage service. */ setup_logger(); - auto parser = setup_argparser(); - + argparse::ArgumentParser parser("Modyn Storage"); + parser.add_argument("config").help("Modyn infrastructure configuration file"); parser.parse_args(argc, argv); - const std::string config_file = parser.get("config"); + const auto config_file = parser.get("config"); ASSERT(std::filesystem::exists(config_file), "Config file does not exist.");