[CI] Stop relying on system BerkeleyDB packages for native builds

.github/workflows/build-and-test.yml

@@ -152,32 +152,32 @@ jobs:
           - name: Linux
             os: ubuntu-22.04
             python-version: '3.8'
-            packages: python3-zmq qttools5-dev qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
+            packages: python3-zmq qttools5-dev qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
             cc: gcc
             cxx: g++
 
           - name: Linux-latest
             os: ubuntu-24.04
             python-version: '3.8'
-            packages: python3-zmq qttools5-dev qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system1.74-dev libboost-filesystem1.74-dev libboost-chrono1.74-dev libboost-test1.74-dev libboost-thread1.74-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
+            packages: python3-zmq qttools5-dev qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system1.74-dev libboost-filesystem1.74-dev libboost-chrono1.74-dev libboost-test1.74-dev libboost-thread1.74-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
             cc: gcc
             cxx: g++
 
           - name: macOS
             os: macos-13
             python-version: '3.10'
-            packages: llvm@14 autoconf automake berkeley-db@4 libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium
+            packages: autoconf automake libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium
             boost_root: true
-            cc: $(brew --prefix llvm@14)/bin/clang
-            cxx: $(brew --prefix llvm@14)/bin/clang++
+            cc: clang
+            cxx: clang++
 
           - name: macOS-latest
             os: macos-14
             python-version: '3.12'
-            packages: llvm@14 autoconf automake berkeley-db@4 libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium
+            packages: autoconf automake libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium
             boost_root: true
-            cc: $(brew --prefix llvm@14)/bin/clang
-            cxx: $(brew --prefix llvm@14)/bin/clang++
+            cc: clang
+            cxx: clang++
 
     steps:
       - name: Get Source
@@ -195,9 +195,13 @@ jobs:
             sudo apt-get install --no-install-recommends --no-upgrade -qq "$APT_BASE" ${{ matrix.config.packages }}
           fi
           if [[ ${{ matrix.config.os }} = macos* ]]; then
-            brew install "$APT_BASE" ${{ matrix.config.packages }}
+            brew install -q "$APT_BASE" ${{ matrix.config.packages }}
           fi
 
+      - name: Install BerkeleyDB
+        run: |
+          sudo ./contrib/install_db4.sh /usr/local true
+
       - name: Setup cmake
         uses: jwlawson/actions-setup-cmake@v2
         with:
@@ -227,18 +231,16 @@ jobs:
             fi
             if [ "${{ matrix.config.boost_root }}" = "true" ]; then
               export CI_BOOST=1
-              if [ "${{ matrix.config.os }}" = "macos-14" ]; then
-                export LDFLAGS="-L/opt/homebrew/opt/boost/lib"
-                export CPPFLAGS="-I/opt/homebrew/opt/boost/include"
-              else
-                export CPPFLAGS="-I/usr/local/opt/boost/include -I/usr/local/opt/berkeley-db@4/include"
-                export LDFLAGS="-L/usr/local/opt/boost/lib -L/usr/local/opt/berkeley-db@4/lib"
-              fi
             fi
           fi
           export CC
           export CXX
           export PYTHON
+          # workaround for macos runners not being able to properly test the native clang++ from within cmake
+          if [[ ${{ matrix.config.os }} = macos* ]]; then
+            ./autogen.sh
+            ./configure
+          fi
           mkdir -p ${{ github.workspace }}/cmake-build-debug && cd ${{ github.workspace }}/cmake-build-debug
           cmake -DCMAKE_BUILD_TYPE=Debug -G "Unix Makefiles" ${{ github.workspace }}
 
@@ -271,27 +273,27 @@ jobs:
             id: Linux-x86_64-nodepends
             os: ubuntu-22.04
             python-version: '3.8'
-            packages: python3-zmq qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
+            packages: python3-zmq qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
             unit_tests: true
             functional_tests: true
             goal: install
-            BITCOIN_CONFIG: "--enable-zmq --enable-debug --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER'"
+            BITCOIN_CONFIG: "--enable-zmq --enable-debug --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER'"
 
           - name: x64-Linux-latest
             id: Linux-x86_64-nodepends-latest
             os: ubuntu-24.04
             python-version: '3.8'
-            packages: python3-zmq qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system1.74-dev libboost-filesystem1.74-dev libboost-chrono1.74-dev libboost-test1.74-dev libboost-thread1.74-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
+            packages: python3-zmq qtbase5-dev qttools5-dev-tools libqt5svg5-dev libqt5charts5-dev libevent-dev bsdmainutils libboost-system1.74-dev libboost-filesystem1.74-dev libboost-chrono1.74-dev libboost-test1.74-dev libboost-thread1.74-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev cargo
             unit_tests: true
             functional_tests: true
             goal: install
-            BITCOIN_CONFIG: "--enable-zmq --enable-debug --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER'"
+            BITCOIN_CONFIG: "--enable-zmq --enable-debug --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER'"
 
           - name: x64-macOS
             id: macOS-nodepends
             os: macos-13
             python-version: '3.10'
-            packages: autoconf automake ccache berkeley-db@4 libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium librsvg
+            packages: autoconf automake ccache libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium librsvg
             unit_tests: true
             functional_tests: true
             goal: deploy
@@ -303,7 +305,7 @@ jobs:
             id: macOS-nodepends-latest
             os: macos-14
             python-version: '3.12'
-            packages: autoconf automake ccache berkeley-db@4 libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium librsvg
+            packages: autoconf automake ccache libtool boost miniupnpc libnatpmp pkg-config qt@5 zmq libevent qrencode gmp libsodium librsvg
             unit_tests: true
             functional_tests: true
             goal: deploy
@@ -327,10 +329,14 @@ jobs:
             sudo apt-get install --no-install-recommends --no-upgrade -qq "$APT_BASE" ${{ matrix.config.packages }}
           fi
           if [[ ${{ matrix.config.os }} = macos* ]]; then
-            brew install ${{ matrix.config.packages }}
+            brew install -q ${{ matrix.config.packages }}
             pip install ds_store mac_alias
           fi
 
+      - name: Install BerkeleyDB
+        run: |
+          sudo ./contrib/install_db4.sh /usr/local true
+
       - name: ccache cache files
         uses: actions/cache@v4
         with:
@@ -438,25 +444,25 @@ jobs:
             id: Linux-x86_64-nodepends-tests
             os: ubuntu-22.04
             python-version: '3.8'
-            packages: python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev
+            packages: python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev
 
           - name: x64-Linux-latest
             id: Linux-x86_64-nodepends-latest-tests
             os: ubuntu-24.04
             python-version: '3.8'
-            packages: python3-zmq libevent-dev bsdmainutils libboost-system1.74-dev libboost-filesystem1.74-dev libboost-chrono1.74-dev libboost-test1.74-dev libboost-thread1.74-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev
+            packages: python3-zmq libevent-dev bsdmainutils libboost-system1.74-dev libboost-filesystem1.74-dev libboost-chrono1.74-dev libboost-test1.74-dev libboost-thread1.74-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libgmp-dev libsodium-dev
 
           - name: x64-macOS
             id: macOS-nodepends
             os: macos-13
             python-version: '3.10'
-            packages: berkeley-db@4 boost miniupnpc libnatpmp pkg-config zmq libevent qrencode gmp libsodium
+            packages: boost miniupnpc libnatpmp pkg-config zmq libevent qrencode gmp libsodium
 
           - name: arm64-macOS-latest
             id: macOS-nodepends-latest
             os: macos-14
             python-version: '3.12'
-            packages: berkeley-db@4 boost miniupnpc libnatpmp pkg-config zmq libevent qrencode gmp libsodium
+            packages: boost miniupnpc libnatpmp pkg-config zmq libevent qrencode gmp libsodium
 
     steps:
       - name: Initialize Python
@@ -471,7 +477,7 @@ jobs:
             sudo apt-get install --no-install-recommends --no-upgrade -qq ${{ matrix.config.packages }}
           fi
           if [[ ${{ matrix.config.os }} = macos* ]]; then
-            brew install ${{ matrix.config.packages }}
+            brew install -q ${{ matrix.config.packages }}
           fi
           pip install pyzmq
 

CMakeLists.txt

@@ -39,13 +39,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
     add_definitions("-DMAC_OSX")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D'NS_FORMAT_ARGUMENT\(A\)='")
     list(APPEND CMAKE_PREFIX_PATH /usr/local/opt/qt5 /usr/local/opt/qt@5 /opt/homebrew/opt/qt5 /opt/homebrew/opt/qt@5)
-    list(APPEND CMAKE_PREFIX_PATH /usr/local/Cellar/berkeley-db@4 /opt/homebrew/Cellar/berkeley-db@4)
-    # Homebrew default path for apple silicon CPUs is different than for intel CPUs
-    if(${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "arm")
-        set(BerkeleyDB_ROOT_DIR "/opt/homebrew/Cellar/berkeley-db@4/${BDB_VER}/")
-    else()
-        set(BerkeleyDB_ROOT_DIR "/usr/local/Cellar/berkeley-db@4/${BDB_VER}/" )
-    endif()
+    list(APPEND CMAKE_PREFIX_PATH /usr/local/Cellar/berkeley-db@4/${BDB_VER} /opt/homebrew/Cellar/berkeley-db@4/${BDB_VER})
     # Github Actions specific Boost override (macOS only)
     if(DEFINED ENV{CI_BOOST})
         if(${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "arm")

configure.ac

@@ -570,12 +570,13 @@ case $host in
          dnl It's safe to add these paths even if the functionality is disabled by
          dnl the user (--without-wallet or --without-gui for example).
 
-         bdb_prefix=`$BREW --prefix berkeley-db4 2>/dev/null`
          qt5_prefix=`$BREW --prefix qt5 2>/dev/null`
          gmp_prefix=`$BREW --prefix gmp 2>/dev/null`
-         if test x$bdb_prefix != x; then
-           CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include"
-           LIBS="$LIBS -L$bdb_prefix/lib"
+         if $BREW list --versions berkeley-db@4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then
+           bdb_prefix=$($BREW --prefix berkeley-db@4 2>/dev/null)
+           dnl This must precede the call to BITCOIN_FIND_BDB48 below.
+           BDB_CFLAGS="-I$bdb_prefix/include"
+           BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8"
          fi
          if test x$qt5_prefix != x; then
            PKG_CONFIG_PATH="$qt5_prefix/lib/pkgconfig:$PKG_CONFIG_PATH"

contrib/cmake/FindBerkeleyDB.cmake

@@ -29,6 +29,8 @@ else()
     list(APPEND _BERKELEYDB_PATHS
             "/usr/local/Cellar/berkeley-db@4"
             "/usr/local/Cellar/berkeley-db"
+            "/opt/homebrew/Cellar/berkeley-db@4"
+            "/opt/homebrew/Cellar/berkeley-db"
             "/opt"
             "/opt/local"
             "/usr/local"

contrib/install_db4.sh

@@ -6,7 +6,7 @@ export LC_ALL=C
 set -e
 
 if [ -z "${1}" ]; then
-  echo "Usage: $0 <base-dir> [<extra-bdb-configure-flag> ...]"
+  echo "Usage: $0 <base-dir> [<ci-mode>] [<extra-bdb-configure-flag> ...]"
   echo
   echo "Must specify a single argument: the directory in which db4 will be built."
   echo "This is probably \`pwd\` if you're at the root of the pivx repository."
@@ -17,7 +17,15 @@ expand_path() {
   cd "${1}" && pwd -P
 }
 
-BDB_PREFIX="$(expand_path ${1})/db4"; shift;
+if [ "$#" -ge 2 ]; then
+  if [ "${2}" != 'true' ]; then
+    BDB_PREFIX="$(expand_path ${1})/db4"; shift;
+    echo "Installing in user mode..."
+  else
+    BDB_PREFIX="$(expand_path ${1})"; shift; shift;
+    echo "Installing in CI mode..."
+  fi
+fi
 BDB_VERSION='db-4.8.30.NC'
 BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef'
 BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz"
@@ -61,7 +69,7 @@ http_get() {
 
 mkdir -p "${BDB_PREFIX}"
 http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}"
-tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX"
+tar -xzf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX"
 cd "${BDB_PREFIX}/${BDB_VERSION}/"
 
 # Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592)
@@ -70,13 +78,18 @@ CLANG_CXX11_PATCH_HASH='7a9a47b03fd5fb93a16ef42235fa9512db9b0829cfc3bdf90edd3ec1
 http_get "${CLANG_CXX11_PATCH_URL}" clang.patch "${CLANG_CXX11_PATCH_HASH}"
 patch -p2 < clang.patch
 
+BIGSUR_PATCH_URL='https://gist.githubusercontent.com/Fuzzbawls/51f8cc35e03d18f35b82e7b36e66491d/raw/d57f6731def206db65191932f5ae7016a325fde5/configure-pre-0.4.2.418-big_sur.diff'
+BIGSUR_PATCH_HASH='83af02f2aa2b746bb7225872cab29a253264be49db0ecebb12f841562d9a2923'
+http_get "${BIGSUR_PATCH_URL}" bigsur.patch "${BIGSUR_PATCH_HASH}"
+patch dist/configure < bigsur.patch
+
 # The packaged config.guess and config.sub are ancient (2009) and can cause build issues.
 # Replace them with modern versions.
 # See https://github.com/bitcoin/bitcoin/issues/16064
-CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b'
-CONFIG_GUESS_HASH='2d1ff7bca773d2ec3c6217118129220fa72d8adda67c7d2bf79994b3129232c1'
-CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b'
-CONFIG_SUB_HASH='3a4befde9bcdf0fdb2763fc1bfa74e8696df94e1ad7aac8042d133c8ff1d2e32'
+CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=00b15927496058d23e6258a28d8996f87cf1f191'
+CONFIG_GUESS_HASH='e3d148130e9151735f8b9a8e69a70d06890ece51468a9762eb7ac0feddddcc2f'
+CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=00b15927496058d23e6258a28d8996f87cf1f191'
+CONFIG_SUB_HASH='11c54f55c3ac99e5d2c3dc2bb0bcccbf69f8223cc68f6b2438daa806cf0d16d8'
 
 rm -f "dist/config.guess"
 rm -f "dist/config.sub"
@@ -86,7 +99,7 @@ http_get "${CONFIG_SUB_URL}" dist/config.sub "${CONFIG_SUB_HASH}"
 
 cd build_unix/
 
-"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \
+CFLAGS="-Wno-error=implicit-function-declaration" "${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \
   --enable-cxx --disable-shared --disable-replication --with-pic --prefix="${BDB_PREFIX}" \
   "${@}"