diff --git a/.github/workflows/linux-build-base.yml b/.github/workflows/linux-build-base.yml index e349fcbf69e3..01ab4bae950d 100644 --- a/.github/workflows/linux-build-base.yml +++ b/.github/workflows/linux-build-base.yml @@ -29,7 +29,7 @@ jobs: # prevent errors when forks ff their main branch if: ${{ github.repository == 'facebookincubator/velox' }} runs-on: 8-core-ubuntu - container: ghcr.io/facebookincubator/velox-dev:adapters + container: ghcr.io/assignuser/velox-dev:adapters defaults: run: shell: bash @@ -95,6 +95,7 @@ jobs: "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" "-DVELOX_ENABLE_GPU=ON" "-DVELOX_MONO_LIBRARY=ON" + "-DVELOX_BUILD_SHARED=ON" ) if [[ "${USE_CLANG}" = "true" ]]; then scripts/setup-centos9.sh install_clang15; export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; CUDA_FLAGS="-ccbin /usr/lib64/llvm15/bin/clang++-15"; fi make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}" diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c56baff117c7..e40ec0b4173a 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -94,6 +94,7 @@ jobs: -DENABLE_ALL_WARNINGS=1 \ -DVELOX_ENABLE_PARQUET=ON \ -DVELOX_MONO_LIBRARY=ON \ + -DVELOX_BUILD_SHARED=ON \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE - name: Build diff --git a/CMake/VeloxUtils.cmake b/CMake/VeloxUtils.cmake index fa8f9456d7e2..aceb9b19175c 100644 --- a/CMake/VeloxUtils.cmake +++ b/CMake/VeloxUtils.cmake @@ -70,8 +70,12 @@ function(velox_add_library TARGET) # Target already exists, append sources to it. target_sources(velox PRIVATE ${ARGN}) else() + set(_type STATIC) + if(VELOX_BUILD_SHARED) + set(_type SHARED) + endif() # Create the target if this is the first invocation. - add_library(velox ${ARGN}) + add_library(velox ${_type} ${ARGN}) set_target_properties(velox PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set_target_properties(velox PROPERTIES ARCHIVE_OUTPUT_DIRECTORY @@ -93,13 +97,26 @@ function(velox_link_libraries TARGET) # TODO(assignUser): Handle scope keywords (they currently are empty calls ala # target_link_libraries(target PRIVATE)) if(VELOX_MONO_LIBRARY) - message(DEBUG "${TARGET}: ${ARGN}") - foreach(_lib ${ARGN}) - if("${_lib}" MATCHES "^velox_*") - message(DEBUG "\t\tDROP: ${_lib}") + # These targets follow the velox_* name for consistency but are NOT actually + # aliases to velox when building the mono lib and need to be linked + # explicitly (this is a hack) + set(explicit_targets + velox_exec_test_lib + # see velox/experimental/wave/README.md + velox_wave_common + velox_wave_decode + velox_wave_dwio + velox_wave_exec + velox_wave_stream + velox_wave_vector) + + foreach(_arg ${ARGN}) + list(FIND explicit_targets ${_arg} _explicit) + if(_explicit EQUAL -1 AND "${_arg}" MATCHES "^velox_*") + message(DEBUG "\t\tDROP: ${_arg}") else() - message(DEBUG "\t\tADDING: ${_lib}") - target_link_libraries(velox ${_lib}) + message(DEBUG "\t\tADDING: ${_arg}") + target_link_libraries(velox ${_arg}) endif() endforeach() else() diff --git a/CMake/resolve_dependency_modules/cpr.cmake b/CMake/resolve_dependency_modules/cpr.cmake index 7f7013ba5de2..488595e08512 100644 --- a/CMake/resolve_dependency_modules/cpr.cmake +++ b/CMake/resolve_dependency_modules/cpr.cmake @@ -35,7 +35,7 @@ FetchContent_Declare( PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/cpr/cpr-libcurl-compatible.patch && git apply ${CMAKE_CURRENT_LIST_DIR}/cpr/cpr-remove-sancheck.patch) -set(BUILD_SHARED_LIBS OFF) +set(BUILD_SHARED_LIBS ${VELOX_BUILD_SHARED}) set(CPR_USE_SYSTEM_CURL OFF) # ZLIB has already been found by find_package(ZLIB, REQUIRED), set CURL_ZLIB=OFF # to save compile time. @@ -44,3 +44,4 @@ FetchContent_MakeAvailable(cpr) # libcpr in its CMakeLists.txt file disables the BUILD_TESTING globally when # CPR_USE_SYSTEM_CURL=OFF. unset BUILD_TESTING here. unset(BUILD_TESTING) +unset(BUILD_SHARED_LIBS) diff --git a/CMake/resolve_dependency_modules/folly/CMakeLists.txt b/CMake/resolve_dependency_modules/folly/CMakeLists.txt index 2f960fbaadab..d501ed8fec2a 100644 --- a/CMake/resolve_dependency_modules/folly/CMakeLists.txt +++ b/CMake/resolve_dependency_modules/folly/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. project(Folly) -cmake_minimum_required(VERSION 3.14) +cmake_minimum_required(VERSION 3.28) set(VELOX_FOLLY_BUILD_VERSION v2024.07.01.00) set(VELOX_FOLLY_BUILD_SHA256_CHECKSUM @@ -27,6 +27,11 @@ message(STATUS "Building Folly from source") if(gflags_SOURCE STREQUAL "BUNDLED") set(glog_patch && git apply ${CMAKE_CURRENT_LIST_DIR}/folly-gflags-glog.patch) + # Together with the patch applied above prevents folly from test compiling a + # snippet to find the right namespace (which would fail because gflags isn't + # built yet) + set(FOLLY_UNUSUAL_GFLAGS_NAMESPACE OFF) + set(FOLLY_GFLAGS_NAMESPACE gflags) endif() FetchContent_Declare( @@ -34,30 +39,19 @@ FetchContent_Declare( URL ${VELOX_FOLLY_SOURCE_URL} URL_HASH ${VELOX_FOLLY_BUILD_SHA256_CHECKSUM} PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/folly-no-export.patch - ${glog_patch}) + ${glog_patch} OVERRIDE_FIND_PACKAGE SYSTEM EXCLUDE_FROM_ALL) + +set(BUILD_SHARED_LIBS ${VELOX_BUILD_SHARED}) -# Suppress all warnings -set(FOLLY_CXX_FLAGS -w) # Enable INT128 support set(FOLLY_HAVE_INT128_T ON) + FetchContent_MakeAvailable(folly) # Folly::folly is not valid for FC but we want to match FindFolly add_library(Folly::folly ALIAS folly) add_library(Folly::follybenchmark ALIAS follybenchmark) -# The folly target does not contain any include directories, they are propagated -# from folly_base. This marks them as system headers which should suppress -# warnigs generated by them when they are included else where. -set_target_properties( - folly_deps - PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES - $) -set_target_properties( - folly_base - PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES - $) - -if(${gflags_SOURCE} STREQUAL "BUNDLED") - add_dependencies(folly glog gflags_static fmt::fmt) +if(gflags_SOURCE STREQUAL "BUNDLED") + add_dependencies(folly glog::glog gflags::gflags fmt::fmt) endif() diff --git a/CMake/resolve_dependency_modules/folly/folly-gflags-glog.patch b/CMake/resolve_dependency_modules/folly/folly-gflags-glog.patch index 6ef25f5ac231..86694ef75aee 100644 --- a/CMake/resolve_dependency_modules/folly/folly-gflags-glog.patch +++ b/CMake/resolve_dependency_modules/folly/folly-gflags-glog.patch @@ -11,6 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +--- a/CMake/FollyConfigChecks.cmake ++++ b/CMake/FollyConfigChecks.cmake +@@ -181,7 +181,7 @@ check_cxx_source_runs(" + HAVE_VSNPRINTF_ERRORS + ) + +-if (FOLLY_HAVE_LIBGFLAGS) ++if (FOLLY_HAVE_LIBGFLAGS AND NOT FOLLY_GFLAGS_NAMESPACE) + # Older releases of gflags used the namespace "gflags"; newer releases + # use "google" but also make symbols available in the deprecated "gflags" + # namespace too. The folly code internally uses "gflags" unless we tell it --- a/CMake/folly-deps.cmake +++ b/CMake/folly-deps.cmake @@ -52,19 +52,20 @@ find_package(DoubleConversion MODULE REQUIRED) @@ -41,7 +52,7 @@ +set(FOLLY_HAVE_LIBGLOG ${glog_FOUND}) +list(APPEND FOLLY_LINK_LIBRARIES ${glog_LIBRARY}) +list(APPEND FOLLY_INCLUDE_DIRECTORIES ${glog_INCLUDE_DIR}) -+message(STATUS "glog_INCLUDE_DIR: ${gflags_LINRARY}") ++message(STATUS "glog_LIBRARY: ${glog_LIBRARY}") find_package(LibEvent MODULE REQUIRED) list(APPEND FOLLY_LINK_LIBRARIES ${LIBEVENT_LIB}) diff --git a/CMake/resolve_dependency_modules/gflags.cmake b/CMake/resolve_dependency_modules/gflags.cmake index 479996b9d124..0a88ea5abb03 100644 --- a/CMake/resolve_dependency_modules/gflags.cmake +++ b/CMake/resolve_dependency_modules/gflags.cmake @@ -28,24 +28,25 @@ FetchContent_Declare( gflags URL ${VELOX_GFLAGS_SOURCE_URL} URL_HASH ${VELOX_GFLAGS_BUILD_SHA256_CHECKSUM} - PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/gflags/gflags-config.patch) + PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/gflags/gflags-config.patch + OVERRIDE_FIND_PACKAGE EXCLUDE_FROM_ALL SYSTEM) -set(GFLAGS_BUILD_STATIC_LIBS ON) -set(GFLAGS_BUILD_gflags_LIB ON) -set(GFLAGS_BUILD_gflags_nothreads_LIB ON) -set(GFLAGS_IS_SUBPROJECT ON) # glog relies on the old `google` namespace set(GFLAGS_NAMESPACE "google;gflags") -FetchContent_MakeAvailable(gflags) +set(GFLAGS_BUILD_SHARED_LIBS ${VELOX_BUILD_SHARED}) +set(GFLAGS_BUILD_STATIC_LIBS ${VELOX_BUILD_STATIC}) -# the flag has to be added to each target we build so adjust to settings choosen -# above -target_compile_options(gflags_static PRIVATE -Wno-cast-function-type) -target_compile_options(gflags_nothreads_static PRIVATE -Wno-cast-function-type) +set(GFLAGS_BUILD_gflags_LIB ON) +set(GFLAGS_BUILD_gflags_nothreads_LIB ON) +set(GFLAGS_IS_SUBPROJECT ON) -# this causes find_package(gflags) to search in the build directory and prevents -# the system gflags from being found -set(gflags_DIR ${gflags_BINARY_DIR}) -set(gflags_LIBRARY gflags_static) -set(gflags_INCLUDE_DIR ${gflags_BINARY_DIR}/include) +# Workaround for https://github.com/gflags/gflags/issues/277 +unset(BUILD_SHARED_LIBS) +FetchContent_MakeAvailable(gflags) +# This causes find_package(gflags) in other dependencies to search in the build +# directory and prevents the system gflags from being found when they don't use +# the target directly (like folly). +set(gflags_FOUND TRUE) +set(gflags_LIBRARY gflags::gflags) +set(gflags_INCLUDE_DIR) diff --git a/CMake/resolve_dependency_modules/glog.cmake b/CMake/resolve_dependency_modules/glog.cmake index 24422bacbf9c..61dfe479e71b 100644 --- a/CMake/resolve_dependency_modules/glog.cmake +++ b/CMake/resolve_dependency_modules/glog.cmake @@ -27,24 +27,36 @@ FetchContent_Declare( glog URL ${VELOX_GLOG_SOURCE_URL} URL_HASH ${VELOX_GLOG_BUILD_SHA256_CHECKSUM} - PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/glog/glog-no-export.patch - && git apply ${CMAKE_CURRENT_LIST_DIR}/glog/glog-config.patch) + PATCH_COMMAND + git apply ${CMAKE_CURRENT_LIST_DIR}/glog/glog-no-export.patch && git apply + ${CMAKE_CURRENT_LIST_DIR}/glog/glog-config.patch SYSTEM + OVERRIDE_FIND_PACKAGE EXCLUDE_FROM_ALL) -set(BUILD_SHARED_LIBS OFF) +set(BUILD_SHARED_LIBS ${VELOX_BUILD_SHARED}) set(WITH_UNWIND OFF) set(gflags_NAMESPACE google) set(BUILD_TESTING OFF) FetchContent_MakeAvailable(glog) unset(BUILD_TESTING) unset(BUILD_SHARED_LIBS) -add_dependencies(glog gflags_static) -list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/glog) -set(glog_INCLUDE_DIR ${glog_BINARY_DIR}) -set(glog_LIBRARY ${glog_BINARY_DIR}/libglog$<$:d>.a) +# Folly uses variables instead of targets +set(glog_LIBRARY glog::glog) -# These headers are missing from the include dir but adding the src dir causes -# issues with folly so we just copy it to the include dir +add_dependencies(glog gflags::gflags) + +# The default target has the glog-src as an include dir but this causes issues +# with folly due to an internal glog 'demangle.h' being mistaken for a system +# header so we remove glog_SOURCE_DIR by overwriting +# INTERFACE_INCLUDE_DIRECTORIES +get_target_property( + _glog_target glog::glog ALIASED_TARGET) # Can't set properties on ALIAS + # targets +set_target_properties( + ${_glog_target} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${glog_BINARY_DIR}) + +# These headers are missing from glog_BINARY_DIR file(COPY ${glog_SOURCE_DIR}/src/glog/platform.h DESTINATION ${glog_BINARY_DIR}/glog) file(COPY ${glog_SOURCE_DIR}/src/glog/log_severity.h diff --git a/CMake/resolve_dependency_modules/gtest.cmake b/CMake/resolve_dependency_modules/gtest.cmake index c8250e58b214..bfbac66d9613 100644 --- a/CMake/resolve_dependency_modules/gtest.cmake +++ b/CMake/resolve_dependency_modules/gtest.cmake @@ -26,7 +26,8 @@ message(STATUS "Building gtest from source") FetchContent_Declare( gtest URL ${VELOX_GTEST_SOURCE_URL} - URL_HASH ${VELOX_GTEST_BUILD_SHA256_CHECKSUM}) + URL_HASH ${VELOX_GTEST_BUILD_SHA256_CHECKSUM} + OVERRIDE_FIND_PACKAGE SYSTEM EXCLUDE_FROM_ALL) FetchContent_MakeAvailable(gtest) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5929473aeb7b..9de9f5a08e48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,7 @@ list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/CMake" # Include our ThirdPartyToolchain dependencies macros include(ResolveDependency) include(VeloxUtils) +include(CMakeDependentOption) velox_set_with_default(VELOX_DEPENDENCY_SOURCE_DEFAULT VELOX_DEPENDENCY_SOURCE AUTO) @@ -77,6 +78,28 @@ option( OFF) option(VELOX_MONO_LIBRARY "Build single unified library." OFF) option(ENABLE_ALL_WARNINGS "Enable -Wall and -Wextra compiler warnings." ON) +option(VELOX_BUILD_SHARED "Build Velox as shared libraries." OFF) +# While it's possible to build both in one go we currently want to build either +# static or shared. +cmake_dependent_option( + VELOX_BUILD_STATIC + "Build Velox as static libraries." + ON + "NOT VELOX_BUILD_SHARED" + OFF) + +if(VELOX_BUILD_SHARED AND NOT VELOX_MONO_LIBRARY) + # The large number of targets currently in use within Velox make a shared + # build when not using the mono library brittle and labor intensive + set(VELOX_MONO_LIBRARY ON) +endif() + +if(VELOX_BUILD_SHARED) + message( + WARNING + "When building Velox as a shared library it's recommended to build against a shared build of folly to avoid issues with linking of gflags." + "This is currently NOT being enforced so user discretion is advised.") +endif() # option() always creates a BOOL variable so we have to use a normal cache # variable with STRING type for this option. diff --git a/scripts/centos.dockerfile b/scripts/centos.dockerfile index 50a18e9a7243..94354e2c673e 100644 --- a/scripts/centos.dockerfile +++ b/scripts/centos.dockerfile @@ -17,6 +17,9 @@ FROM $image COPY scripts/setup-helper-functions.sh / COPY scripts/setup-centos9.sh / + +# Building libvelox.so requires folly and gflags to be built shared as well for now +ENV VELOX_BUILD_SHARED=ON # The removal of the build dir has to happen in the same layer as the build # to minimize the image size. gh & jq are required for CI RUN mkdir build && ( cd build && bash /setup-centos9.sh ) && rm -rf build && \ diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh index 8ecb07289901..2f28f31e414d 100755 --- a/scripts/setup-adapters.sh +++ b/scripts/setup-adapters.sh @@ -151,16 +151,7 @@ function install_azure-storage-sdk-cpp { } function install_hdfs_deps { - github_checkout apache/hawq master - libhdfs3_dir=hawq/depends/libhdfs3 - if [[ "$OSTYPE" == darwin* ]]; then - sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $DEPENDENCY_DIR/$libhdfs3_dir/CMakeLists.txt - sed -i '' -e "s/dumpversion/dumpfullversion/" $DEPENDENCY_DIR/$libhdfs3_dir/CMakeLists.txt - fi - if [[ "$OSTYPE" == linux-gnu* ]]; then - sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $DEPENDENCY_DIR/$libhdfs3_dir/CMakeLists.txt - sed -i "s/dumpversion/dumpfullversion/" $DEPENDENCY_DIR/$libhdfs3_dir/CMake/Platform.cmake # Dependencies for Hadoop testing wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0.tar.gz hadoop cp -a ${DEPENDENCY_DIR}/hadoop /usr/local/ @@ -173,7 +164,6 @@ function install_hdfs_deps { yum install -y java-1.8.0-openjdk-devel fi fi - cmake_install_dir $libhdfs3_dir } (mkdir -p "${DEPENDENCY_DIR}") || exit diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 1cf43630c4f1..a75bbb02596d 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -34,6 +34,7 @@ NPROC=$(getconf _NPROCESSORS_ONLN) export CXXFLAGS=$(get_cxx_flags) # Used by boost. export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO. CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so. BUILD_DUCKDB="${BUILD_DUCKDB:-true}" USE_CLANG="${USE_CLANG:-false}" export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} @@ -89,7 +90,7 @@ function install_gflags { # Remove an older version if present. dnf remove -y gflags wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags - cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 + cmake_install_dir gflags -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 } function install_glog { @@ -153,7 +154,7 @@ function install_fizz { function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - cmake_install_dir folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + cmake_install_dir folly -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } function install_wangle { diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index afccef5ecaf9..efc3ab0f5621 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -39,6 +39,7 @@ export OS_CXXFLAGS=" -isystem $(brew --prefix)/include " NPROC=$(getconf _NPROCESSORS_ONLN) BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so. DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} MACOS_VELOX_DEPS="bison flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 snappy xz zstd" MACOS_BUILD_DEPS="ninja cmake" @@ -116,7 +117,7 @@ function install_fmt { function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - cmake_install_dir folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + cmake_install_dir folly -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DFOLLY_HAVE_INT128_T=ON } function install_fizz { diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 3d502547a7f6..7576fde00f16 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -37,6 +37,7 @@ export COMPILER_FLAGS NPROC=$(getconf _NPROCESSORS_ONLN) BUILD_DUCKDB="${BUILD_DUCKDB:-true}" export CMAKE_BUILD_TYPE=Release +VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly shared for use in libvelox.so. SUDO="${SUDO:-"sudo --preserve-env"}" USE_CLANG="${USE_CLANG:-false}" export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} @@ -169,7 +170,7 @@ function install_protobuf { function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - cmake_install_dir folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + cmake_install_dir folly -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DFOLLY_HAVE_INT128_T=ON } function install_fizz { diff --git a/velox/dwio/dwrf/test/utils/CMakeLists.txt b/velox/dwio/dwrf/test/utils/CMakeLists.txt index 89bfc2e4fdf9..16f035f7670a 100644 --- a/velox/dwio/dwrf/test/utils/CMakeLists.txt +++ b/velox/dwio/dwrf/test/utils/CMakeLists.txt @@ -20,6 +20,7 @@ target_link_libraries( velox_dwio_common velox_dwio_common_exception velox_dwio_dwrf_reader + velox_dwio_common_test_utils velox_dwio_dwrf_writer velox_exception velox_memory diff --git a/velox/dwio/parquet/tests/writer/ParquetWriterTest.cpp b/velox/dwio/parquet/tests/writer/ParquetWriterTest.cpp index 0de6eef55056..c7510d27bff8 100644 --- a/velox/dwio/parquet/tests/writer/ParquetWriterTest.cpp +++ b/velox/dwio/parquet/tests/writer/ParquetWriterTest.cpp @@ -23,8 +23,8 @@ #include "velox/core/QueryCtx.h" #include "velox/dwio/parquet/RegisterParquetWriter.h" // @manual #include "velox/dwio/parquet/tests/ParquetTestBase.h" +#include "velox/exec/Cursor.h" #include "velox/exec/tests/utils/AssertQueryBuilder.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/exec/tests/utils/QueryAssertions.h" #include "velox/exec/tests/utils/TempDirectoryPath.h" diff --git a/velox/exec/CMakeLists.txt b/velox/exec/CMakeLists.txt index 25882fd44f14..80bd83d2084e 100644 --- a/velox/exec/CMakeLists.txt +++ b/velox/exec/CMakeLists.txt @@ -110,6 +110,22 @@ velox_link_libraries( velox_arrow_bridge velox_common_compression) +velox_add_library(velox_cursor Cursor.cpp) +velox_link_libraries( + velox_cursor + velox_core + velox_exception + velox_expression + velox_dwio_common + velox_dwio_dwrf_reader + velox_dwio_dwrf_writer + velox_type_fbhive + velox_hive_connector + velox_tpch_connector + velox_presto_serializer + velox_functions_prestosql + velox_aggregates) + if(${VELOX_BUILD_TESTING}) add_subdirectory(fuzzer) add_subdirectory(tests) diff --git a/velox/exec/tests/utils/Cursor.cpp b/velox/exec/Cursor.cpp similarity index 99% rename from velox/exec/tests/utils/Cursor.cpp rename to velox/exec/Cursor.cpp index eaa098997f15..570c314e70f6 100644 --- a/velox/exec/tests/utils/Cursor.cpp +++ b/velox/exec/Cursor.cpp @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "velox/exec/tests/utils/Cursor.h" +#include "velox/exec/Cursor.h" #include "velox/common/file/FileSystems.h" #include "velox/exec/Operator.h" diff --git a/velox/exec/tests/utils/Cursor.h b/velox/exec/Cursor.h similarity index 100% rename from velox/exec/tests/utils/Cursor.h rename to velox/exec/Cursor.h diff --git a/velox/exec/benchmarks/WindowPrefixSortBenchmark.cpp b/velox/exec/benchmarks/WindowPrefixSortBenchmark.cpp index e762a7f821c2..c91f40a490f6 100644 --- a/velox/exec/benchmarks/WindowPrefixSortBenchmark.cpp +++ b/velox/exec/benchmarks/WindowPrefixSortBenchmark.cpp @@ -19,7 +19,7 @@ #include #include -#include "velox/exec/tests/utils/Cursor.h" +#include "velox/exec/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" diff --git a/velox/exec/tests/DriverTest.cpp b/velox/exec/tests/DriverTest.cpp index 2aea378035ee..c4333312474d 100644 --- a/velox/exec/tests/DriverTest.cpp +++ b/velox/exec/tests/DriverTest.cpp @@ -21,11 +21,11 @@ #include "velox/common/base/tests/GTestUtils.h" #include "velox/common/testutil/TestValue.h" #include "velox/dwio/common/tests/utils/BatchMaker.h" +#include "velox/exec/Cursor.h" #include "velox/exec/PlanNodeStats.h" #include "velox/exec/Values.h" #include "velox/exec/tests/utils/ArbitratorTestUtil.h" #include "velox/exec/tests/utils/AssertQueryBuilder.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/OperatorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/functions/Udf.h" diff --git a/velox/exec/tests/GroupedExecutionTest.cpp b/velox/exec/tests/GroupedExecutionTest.cpp index e2e788506a89..0a7fc85cef71 100644 --- a/velox/exec/tests/GroupedExecutionTest.cpp +++ b/velox/exec/tests/GroupedExecutionTest.cpp @@ -17,9 +17,9 @@ #include "velox/common/base/tests/GTestUtils.h" #include "velox/common/memory/MemoryArbitrator.h" +#include "velox/exec/Cursor.h" #include "velox/exec/OutputBufferManager.h" #include "velox/exec/PlanNodeStats.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/exec/tests/utils/TempDirectoryPath.h" diff --git a/velox/exec/tests/HashJoinTest.cpp b/velox/exec/tests/HashJoinTest.cpp index 7698cecd3aa8..85cea6fab72e 100644 --- a/velox/exec/tests/HashJoinTest.cpp +++ b/velox/exec/tests/HashJoinTest.cpp @@ -21,13 +21,13 @@ #include "velox/common/base/tests/GTestUtils.h" #include "velox/common/testutil/TestValue.h" #include "velox/dwio/common/tests/utils/BatchMaker.h" +#include "velox/exec/Cursor.h" #include "velox/exec/HashBuild.h" #include "velox/exec/HashJoinBridge.h" #include "velox/exec/OperatorUtils.h" #include "velox/exec/PlanNodeStats.h" #include "velox/exec/tests/utils/ArbitratorTestUtil.h" #include "velox/exec/tests/utils/AssertQueryBuilder.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/exec/tests/utils/TempDirectoryPath.h" diff --git a/velox/exec/tests/TableScanTest.cpp b/velox/exec/tests/TableScanTest.cpp index 0ebd1222c988..d8292880caee 100644 --- a/velox/exec/tests/TableScanTest.cpp +++ b/velox/exec/tests/TableScanTest.cpp @@ -34,12 +34,12 @@ #include "velox/connectors/hive/HivePartitionFunction.h" #include "velox/dwio/common/CacheInputStream.h" #include "velox/dwio/common/tests/utils/DataFiles.h" +#include "velox/exec/Cursor.h" #include "velox/exec/Exchange.h" #include "velox/exec/OutputBufferManager.h" #include "velox/exec/PlanNodeStats.h" #include "velox/exec/TableScan.h" #include "velox/exec/tests/utils/AssertQueryBuilder.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/LocalExchangeSource.h" #include "velox/exec/tests/utils/PlanBuilder.h" diff --git a/velox/exec/tests/TaskTest.cpp b/velox/exec/tests/TaskTest.cpp index d78a888a1b2d..b527b7e6b6ce 100644 --- a/velox/exec/tests/TaskTest.cpp +++ b/velox/exec/tests/TaskTest.cpp @@ -24,11 +24,11 @@ #include "velox/common/memory/tests/SharedArbitratorTestUtil.h" #include "velox/common/testutil/TestValue.h" #include "velox/connectors/hive/HiveConnectorSplit.h" +#include "velox/exec/Cursor.h" #include "velox/exec/OutputBufferManager.h" #include "velox/exec/PlanNodeStats.h" #include "velox/exec/Values.h" #include "velox/exec/tests/utils/AssertQueryBuilder.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/exec/tests/utils/QueryAssertions.h" diff --git a/velox/exec/tests/utils/CMakeLists.txt b/velox/exec/tests/utils/CMakeLists.txt index 294493367012..0df50966d54b 100644 --- a/velox/exec/tests/utils/CMakeLists.txt +++ b/velox/exec/tests/utils/CMakeLists.txt @@ -17,22 +17,6 @@ add_library(velox_temp_path TempFilePath.cpp TempDirectoryPath.cpp) target_link_libraries( velox_temp_path velox_exception) -add_library(velox_cursor Cursor.cpp) -target_link_libraries( - velox_cursor - velox_core - velox_exception - velox_expression - velox_dwio_common - velox_dwio_dwrf_reader - velox_dwio_dwrf_writer - velox_type_fbhive - velox_hive_connector - velox_tpch_connector - velox_presto_serializer - velox_functions_prestosql - velox_aggregates) - add_library( velox_exec_test_lib AssertQueryBuilder.cpp @@ -51,6 +35,7 @@ add_library( target_link_libraries( velox_exec_test_lib velox_vector_test_lib + velox_vector_fuzzer velox_temp_path velox_cursor velox_core diff --git a/velox/exec/tests/utils/QueryAssertions.cpp b/velox/exec/tests/utils/QueryAssertions.cpp index c13a9a049c34..2e274da5ec60 100644 --- a/velox/exec/tests/utils/QueryAssertions.cpp +++ b/velox/exec/tests/utils/QueryAssertions.cpp @@ -19,8 +19,8 @@ #include "duckdb/common/types.hpp" // @manual #include "velox/duckdb/conversion/DuckConversion.h" +#include "velox/exec/Cursor.h" #include "velox/exec/tests/utils/ArbitratorTestUtil.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/QueryAssertions.h" #include "velox/functions/prestosql/types/TimestampWithTimeZoneType.h" #include "velox/vector/VectorTypeUtils.h" diff --git a/velox/exec/tests/utils/QueryAssertions.h b/velox/exec/tests/utils/QueryAssertions.h index 81bfc036b7f1..ff81add3a2a8 100644 --- a/velox/exec/tests/utils/QueryAssertions.h +++ b/velox/exec/tests/utils/QueryAssertions.h @@ -18,8 +18,8 @@ #include "velox/common/testutil/TestValue.h" #include "velox/core/PlanNode.h" +#include "velox/exec/Cursor.h" #include "velox/exec/Operator.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/vector/ComplexVector.h" #include // @manual diff --git a/velox/experimental/gpu/tests/CMakeLists.txt b/velox/experimental/gpu/tests/CMakeLists.txt index d4a6a37f6776..6202a78d0bd1 100644 --- a/velox/experimental/gpu/tests/CMakeLists.txt +++ b/velox/experimental/gpu/tests/CMakeLists.txt @@ -11,7 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +list(APPEND CMAKE_PREFIX_PATH "${CUDAToolkit_LIBRARY_DIR}/cmake") +find_package(CUB REQUIRED) add_executable(velox_gpu_hash_table_test HashTableTest.cu) target_link_libraries( - velox_gpu_hash_table_test Folly::folly gflags::gflags) + velox_gpu_hash_table_test + Folly::folly + gflags::gflags + glog::glog + CUB::CUB + CUDA::cudart) diff --git a/velox/experimental/wave/README.md b/velox/experimental/wave/README.md new file mode 100644 index 000000000000..008e48b50f4c --- /dev/null +++ b/velox/experimental/wave/README.md @@ -0,0 +1,33 @@ + + +# CMake: Use Base Functions + +> [!IMPORTANT] Please use `target_link_libraries` and `add_library` +> instead of the `velox_*` functions when adding or linking to targets +> within wave/ and label tests with `cuda_driver`. + +The `wave` GPU component links against the CUDA driver in several targets. +They can be built on machines without the actual driver installed, this +requires the relevant 'stub' packages to be installed (see setup scripts). + +Any library that statically links against the stubs **can not** run on a +machine without an actual CUDA driver installed (like our CI). +For this reason we need to use the base functions to create standalone +libraries for wave to avoid linking statically against the stubs when +building the monolithic library and label any tests with 'cuda_driver' +to allow excluding them from ctest on machines without the driver. + diff --git a/velox/experimental/wave/common/CMakeLists.txt b/velox/experimental/wave/common/CMakeLists.txt index 8cea9f79c1a5..be2d0763161a 100644 --- a/velox/experimental/wave/common/CMakeLists.txt +++ b/velox/experimental/wave/common/CMakeLists.txt @@ -29,8 +29,8 @@ target_link_libraries( velox_common_base velox_type CUDA::cuda_driver - CUDA::cudart - CUDA::nvrtc) + CUDA::nvrtc + CUDA::cudart) if(${VELOX_BUILD_TESTING}) add_subdirectory(tests) diff --git a/velox/experimental/wave/common/tests/CMakeLists.txt b/velox/experimental/wave/common/tests/CMakeLists.txt index 2d797fb89314..c0ed173cf7cc 100644 --- a/velox/experimental/wave/common/tests/CMakeLists.txt +++ b/velox/experimental/wave/common/tests/CMakeLists.txt @@ -31,6 +31,7 @@ target_include_directories(velox_wave_common_test PRIVATE ../../../breeze) target_link_libraries( velox_wave_common_test velox_wave_common + xsimd GTest::gtest GTest::gtest_main CUDA::cudart) diff --git a/velox/experimental/wave/dwio/CMakeLists.txt b/velox/experimental/wave/dwio/CMakeLists.txt index f43052462fdc..192a71f77a7e 100644 --- a/velox/experimental/wave/dwio/CMakeLists.txt +++ b/velox/experimental/wave/dwio/CMakeLists.txt @@ -14,11 +14,8 @@ add_subdirectory(decode) -velox_add_library( - velox_wave_dwio - ColumnReader.cpp - FormatData.cpp - ReadStream.cpp - StructColumnReader.cpp) +add_library(velox_wave_dwio ColumnReader.cpp FormatData.cpp ReadStream.cpp + StructColumnReader.cpp) -velox_link_libraries(velox_wave_dwio Folly::folly fmt::fmt xsimd) +target_link_libraries( + velox_wave_dwio Folly::folly fmt::fmt xsimd) diff --git a/velox/experimental/wave/dwio/decode/CMakeLists.txt b/velox/experimental/wave/dwio/decode/CMakeLists.txt index 6e7e9c780e41..5555ac76977b 100644 --- a/velox/experimental/wave/dwio/decode/CMakeLists.txt +++ b/velox/experimental/wave/dwio/decode/CMakeLists.txt @@ -17,4 +17,4 @@ add_subdirectory(tests) add_library(velox_wave_decode GpuDecoder.cu) target_link_libraries( - velox_wave_decode velox_wave_common) + velox_wave_decode velox_wave_common CUDA::cudart) diff --git a/velox/experimental/wave/exec/CMakeLists.txt b/velox/experimental/wave/exec/CMakeLists.txt index 4b4d7ef2619b..3a5447d6cabd 100644 --- a/velox/experimental/wave/exec/CMakeLists.txt +++ b/velox/experimental/wave/exec/CMakeLists.txt @@ -42,7 +42,8 @@ target_link_libraries( velox_wave_stream velox_exception velox_common_base - velox_exec) + velox_exec + CUDA::cudart) if(${VELOX_BUILD_TESTING}) add_subdirectory(tests) diff --git a/velox/experimental/wave/vector/CMakeLists.txt b/velox/experimental/wave/vector/CMakeLists.txt index 7e3082486789..b70f331e3b11 100644 --- a/velox/experimental/wave/vector/CMakeLists.txt +++ b/velox/experimental/wave/vector/CMakeLists.txt @@ -12,5 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -velox_add_library(velox_wave_vector WaveVector.cpp) -velox_link_libraries(velox_wave_vector velox_vector velox_common_base) +add_library(velox_wave_vector WaveVector.cpp) +target_link_libraries( + velox_wave_vector velox_vector velox_common_base) diff --git a/velox/flag_definitions/CMakeLists.txt b/velox/flag_definitions/CMakeLists.txt index 8569cc3f997d..8b551113d735 100644 --- a/velox/flag_definitions/CMakeLists.txt +++ b/velox/flag_definitions/CMakeLists.txt @@ -11,5 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -velox_add_library(velox_flag_definitions OBJECT flags.cpp) +velox_add_library(velox_flag_definitions flags.cpp) velox_link_libraries(velox_flag_definitions PRIVATE gflags::gflags) diff --git a/velox/functions/prestosql/aggregates/benchmarks/ReduceAgg.cpp b/velox/functions/prestosql/aggregates/benchmarks/ReduceAgg.cpp index 39ef1f99c9d5..4fe8bdef3f84 100644 --- a/velox/functions/prestosql/aggregates/benchmarks/ReduceAgg.cpp +++ b/velox/functions/prestosql/aggregates/benchmarks/ReduceAgg.cpp @@ -17,7 +17,7 @@ #include #include -#include "velox/exec/tests/utils/Cursor.h" +#include "velox/exec/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/vector/fuzzer/VectorFuzzer.h" diff --git a/velox/functions/prestosql/aggregates/benchmarks/SimpleAggregates.cpp b/velox/functions/prestosql/aggregates/benchmarks/SimpleAggregates.cpp index 58edb37da9a2..3018ce78582a 100644 --- a/velox/functions/prestosql/aggregates/benchmarks/SimpleAggregates.cpp +++ b/velox/functions/prestosql/aggregates/benchmarks/SimpleAggregates.cpp @@ -18,7 +18,7 @@ #include #include -#include "velox/exec/tests/utils/Cursor.h" +#include "velox/exec/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/vector/fuzzer/VectorFuzzer.h" diff --git a/velox/functions/prestosql/aggregates/benchmarks/TwoStringKeys.cpp b/velox/functions/prestosql/aggregates/benchmarks/TwoStringKeys.cpp index 8c43c625851e..f3eb9f149a80 100644 --- a/velox/functions/prestosql/aggregates/benchmarks/TwoStringKeys.cpp +++ b/velox/functions/prestosql/aggregates/benchmarks/TwoStringKeys.cpp @@ -17,8 +17,8 @@ #include #include +#include "velox/exec/Cursor.h" #include "velox/exec/PlanNodeStats.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/exec/tests/utils/HiveConnectorTestBase.h" #include "velox/exec/tests/utils/PlanBuilder.h" #include "velox/vector/fuzzer/VectorFuzzer.h" diff --git a/velox/runner/LocalRunner.h b/velox/runner/LocalRunner.h index f5d3bb5c211f..3d69a07da8ab 100644 --- a/velox/runner/LocalRunner.h +++ b/velox/runner/LocalRunner.h @@ -16,8 +16,8 @@ #pragma once #include "velox/connectors/Connector.h" +#include "velox/exec/Cursor.h" #include "velox/exec/Exchange.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/runner/LocalSchema.h" #include "velox/runner/MultiFragmentPlan.h" #include "velox/runner/Runner.h" diff --git a/velox/runner/Runner.h b/velox/runner/Runner.h index e570a0df1cf2..88707e87f948 100644 --- a/velox/runner/Runner.h +++ b/velox/runner/Runner.h @@ -16,8 +16,8 @@ #pragma once #include "velox/connectors/Connector.h" +#include "velox/exec/Cursor.h" #include "velox/exec/Exchange.h" -#include "velox/exec/tests/utils/Cursor.h" #include "velox/runner/LocalSchema.h" #include "velox/runner/MultiFragmentPlan.h" diff --git a/velox/tool/trace/CMakeLists.txt b/velox/tool/trace/CMakeLists.txt index d179b26c340e..d36ee8fab023 100644 --- a/velox/tool/trace/CMakeLists.txt +++ b/velox/tool/trace/CMakeLists.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -velox_add_library( +add_library( velox_query_trace_replayer_base AggregationReplayer.cpp FilterProjectReplayer.cpp @@ -23,7 +23,7 @@ velox_add_library( TableWriterReplayer.cpp TraceReplayRunner.cpp) -velox_link_libraries( +target_link_libraries( velox_query_trace_replayer_base velox_aggregates velox_type