Build 3.0.3-rc04 err with "no matching constructor for initialization of 'lucene::index::IndexWriter'"

[whg517]

env

• os: registry.access.redhat.com/ubi9
• ldb-toolchain: 0.21
• java: 17
• doris: 3.0.3-rc04

build script

FROM registry.access.redhat.com/ubi9

ARG LDB_TOOLCHAIN_VERSION=0.21
ARG PRODUCT_VERSION=3.0.3-rc04
ARG JAVA_VERSION=17

# install system requirements
RUN <<EOT
    # https://adoptium.net/en-GB/installation/linux/#_centosrhelfedora_instructions
    cat <<EOF > /etc/yum.repos.d/adoptium.repo
[Adoptium]
name=Adoptium
baseurl=https://packages.adoptium.net/artifactory/rpm/${DISTRIBUTION_NAME:-$(. /etc/os-release; echo $ID)}/\$releasever/\$basearch
enabled=1
gpgcheck=1
gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public
EOF

    dnf update -y
    dnf install -y \
        automake \
        bzip2 \
        cmake \
        cyrus-sasl-devel \
        diffutils \
        fuse-devel \
        gcc \
        gcc-c++ \
        gettext \
        git \
        gzip \
        hostname \
        krb5-devel \
        libcurl-devel \
        libtool \
        make \
        maven \
        nodejs \
        openssl-devel \
        patch \
        pkg-config \
        perl-CPAN \
        tar \
        temurin-17-jdk \
        tzdata-java \
        unzip \
        wget \
        which \
        xz \
        zlib-devel \
        zip

    microdnf clean all
    rm -rf /var/cache/yum

    # smoke test
    java -version
EOT

ENV JAVA_HOME=/usr/lib/jvm/temurin-${JAVA_VERSION}-jdk
ENV JAVA_VERSION=${JAVA_VERSION}

# setup gettext
# Because of the gettext of dnf is lack autopoint, so we need to install gettext from source.
# Download gettext https://ftp.gnu.org/gnu/gettext/gettext-0.21.tar.gz
RUN <<EOF
    set -ex

    # Get already installed gettext version
    GETTEXT_VERSION=$(gettext --version | head -n 1 | awk '{print $4}')

    mkdir -p /build/gettext
    mkdir -p /opt/gettext
    pushd /build/gettext
    curl -sSLf https://ftp.gnu.org/gnu/gettext/gettext-${GETTEXT_VERSION}.tar.gz \
        | tar -xz --strip-components=1

    ./configure --prefix=/opt/gettext
    make -j$(nproc)
    make install
    popd

    # cleanup
    rm -rf /build/gettext
EOF

ENV PATH="/opt/gettext/bin:$PATH"

# setup ldb-toolchain
# Download ldb_toolchain_gen.aarch64.sh
# https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.9.1/ldb_toolchain_gen.aarch64.sh
# or https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.9.1/ldb_toolchain_gen.sh
# Note:
#   - In 0.9.1, ldb-toolchain containes curl, it conflicts with the system curl,
#     when use it, it will cause dns resolution error
#   - In 0.21, ldb-toolchain containes curl, it will verify the certificate
#     file /etc/ssl/certs/ca-certificates.crt. But ca-certificates package generate a default
#     file /etc/ssl/certs/ca-bundle.crt, so we need to create a symlink to fix it.
#   - 0.14.1 aaarch64 version is error to build doris
RUN <<EOF
    set -ex
    ARCH=$(uname -m)
    ARCH=${ARCH/arm64/aarch64}

    release_file="ldb_toolchain_gen.sh"
    if [ "$ARCH" == "aarch64" ]; then
        release_file="ldb_toolchain_gen.aarch64.sh"
    fi

    mkdir -p /build/ldb-toolchain
    pushd /build/ldb-toolchain
    curl -sSLf https://github.com/amosbird/ldb_toolchain_gen/releases/download/v${LDB_TOOLCHAIN_VERSION}/${release_file} \
        -o ldb_toolchain_gen.sh

    chmod +x ldb_toolchain_gen.sh

    ./ldb_toolchain_gen.sh /opt/ldb-toolchain
    popd

    # create a symlink to bison, because doris need byacc
    ln -s /opt/ldb-toolchain/bin/bison /opt/ldb-toolchain/bin/byacc

    # fix /opt/ldb-toolchain/bin/curl certificate verify failed
    ln -s /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt

    # cleanup
    rm -rf /build/ldb-toolchain
EOF

ENV PATH="/opt/ldb-toolchain/bin:$PATH"

# build doris
# Download doris https://github.com/apache/doris/archive/refs/tags/3.0.3-rc04.tar.gz
RUN <<EOF
    set -ex
    # alias python3 to python
    ln -s /usr/bin/python3 /usr/bin/python

    mkdir -p /build/doris
    pushd /build/doris
    curl -sSLf https://github.com/apache/doris/archive/refs/tags/${PRODUCT_VERSION}.tar.gz \
        | tar -xz --strip-components=1

    # Add `set -x` after `set -euo pipefail`
    sed -i '/set -eo pipefail/a set -x' /build/doris/build.sh
    sed -i '/set -eo pipefail/a set -x' /build/doris/thirdparty/build-thirdparty.sh

    # Patch submodules
    # Patched sha from source by `git ls-remote`
    sed -i "s|refs/heads/orc.tar.gz|db01184f765c03496e4107bd3ac37c077ac4bc5f/orc.tar.gz|g" build.sh
    sed -i "s|refs/heads/clucene.tar.gz|48fa9cc4ec32b40bf3b02338d0a1b2cdbc6408cf/clucene.tar.gz|g" build.sh

    # build doris, disable avx2
    export USE_AVX2=OFF
    export USE_UNWIND=OFF
    # ./build.sh
    nohup ./build.sh > /build/doris-build.log 2>&1 &
    tail -f /build/doris-build.log

    # TODO
EOF

error msg

[832/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/pb_convert.cpp.o
[833/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/cloud_tablet_mgr.cpp.o
[834/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/injection_point_action.cpp.o
[835/1371] Building CXX object src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o
FAILED: src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o
/opt/ldb-toolchain/bin/clang++ -DBOOST_CONTAINER_NO_LIB -DBOOST_DATE_TIME_NO_LIB -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_STACKTRACE_USE_BACKTRACE -DBOOST_SYSTEM_NO_DEPRECATED -DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1 -DBRPC_ENABLE_CPU_PROFILER -DGLOG_CUSTOM_PREFIX_SUPPORT -DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DLIBJVM -DS2_USE_GFLAGS -DS2_USE_GLOG -DUSE_HADOOP_HDFS -DUSE_JEMALLOC -DUSE_MEM_TRACKER -D__STDC_FORMAT_MACROS -Dindex_tool_EXPORTS -I/build/doris/be/src/apache-orc/c++/include -I/build/doris/be/build_Release/src/apache-orc/c++/include -I/build/doris/be/build_Release/src/clucene/src/shared -I/build/doris/be/src/clucene/src/core -I/build/doris/be/src/clucene/src/shared -I/build/doris/be/src/clucene/src/contribs-lib -I/build/doris/be/src -I/build/doris/be/test -I/usr/lib/jvm/temurin-17-jdk/include -I/usr/lib/jvm/temurin-17-jdk/include/linux -I/build/doris/be/../gensrc/build/common -isystem /build/doris/be/../common -isystem /build/doris/be/../gensrc/build -isystem /build/doris/thirdparty/installed/include -isystem /build/doris/thirdparty/installed/gperftools/include -isystem /build/doris/thirdparty/installed/include/lz4 -isystem /build/doris/thirdparty/installed/include/zstd -O3 -DNDEBUG  -O3 -DNDEBUG -std=gnu++20   -D OS_LINUX -g -Wall -Wextra -Werror -pthread -fstrict-aliasing -fno-omit-frame-pointer -Wnon-virtual-dtor -Wno-unused-parameter -Wno-sign-compare -fcolor-diagnostics -Wpedantic -Wshadow-field -Wunused -Wunused-command-line-argument -Wunused-exception-parameter -Wunused-volatile-lvalue -Wunused-template -Wunused-member-function -Wunused-macros -Wconversion -Wno-gnu-statement-expression -Wno-implicit-float-conversion -Wno-implicit-int-conversion -Wno-sign-conversion -Wno-missing-field-initializers -Wno-unused-const-variable -Wno-shorten-64-to-32 -march=armv8-a+crc -Winvalid-pch -Xclang -include-pch -Xclang /build/doris/be/build_Release/CMakeFiles/pch.dir/cmake_pch.hxx.pch -Xclang -include -Xclang /build/doris/be/build_Release/CMakeFiles/pch.dir/cmake_pch.hxx -MD -MT src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o -MF src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o.d -o src/index-tools/CMakeFiles/index_tool.dir/index_tool.cpp.o -c /build/doris/be/src/index-tools/index_tool.cpp
/build/doris/be/src/index-tools/index_tool.cpp:565:35: error: no matching constructor for initialization of 'lucene::index::IndexWriter'
  565 |         auto indexwriter = _CLNEW lucene::index::IndexWriter(dir, analyzer, true, true);
      |                                   ^                          ~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:579:12: note: candidate constructor not viable: no known conversion from 'std::shared_ptr<doris::segment_v2::DorisFSDirectory>' to 'lucene::store::Directory *' for 1st argument
  579 |   explicit IndexWriter(CL_NS(store)::Directory* d, CL_NS(analysis)::Analyzer* a, const bool create, const bool closeDirOnShutdown=false);
      |            ^           ~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:625:12: note: candidate constructor not viable: no known conversion from 'std::shared_ptr<doris::segment_v2::DorisFSDirectory>' to 'lucene::store::Directory *' for 1st argument
  625 |   explicit IndexWriter(CL_NS(store)::Directory* d, const bool autoCommit, CL_NS(analysis)::Analyzer* a, const bool create, IndexDeletionPolicy* deletionPolicy = NULL, const bool closeDirOnShutdown=false);
      |            ^           ~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:599:12: note: candidate constructor not viable: no known conversion from 'std::shared_ptr<doris::segment_v2::DorisFSDirectory>' to 'lucene::store::Directory *' for 1st argument
  599 |   explicit IndexWriter(CL_NS(store)::Directory* d, const bool autoCommit, CL_NS(analysis)::Analyzer* a, IndexDeletionPolicy* deletionPolicy = NULL, const bool closeDirOnShutdown=false);
      |            ^           ~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:557:12: note: candidate constructor not viable: requires 3 arguments, but 4 were provided
  557 |   explicit IndexWriter(const char* path, CL_NS(analysis)::Analyzer* a, const bool create);
      |            ^           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/build/doris/be/src/clucene/src/core/CLucene/index/IndexWriter.h:187:22: note: candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 4 were provided
  187 | class CLUCENE_EXPORT IndexWriter:LUCENE_BASE {
      |                      ^~~~~~~~~~~
1 error generated.
[836/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/cloud_txn_delete_bitmap_cache.cpp.o
[837/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/cloud_warm_up_manager.cpp.o
[838/1371] Building CXX object src/util/CMakeFiles/Util.dir/arrow/block_convertor.cpp.o
[839/1371] Building CXX object src/tools/CMakeFiles/meta_tool.dir/meta_tool.cpp.o
[840/1371] Building CXX object src/olap/CMakeFiles/Olap.dir/delete_handler.cpp.o
[841/1371] Building CXX object src/olap/CMakeFiles/Olap.dir/tablet_reader.cpp.o

files

source code from 3.0.3-rc04

doris/be/src/index-tools/index_tool.cpp

Lines 550 to 566 in 62a58bf

	auto fs = doris::io::global_local_filesystem();
	auto index_file_writer = std::make_unique<InvertedIndexFileWriter>(
	fs,
	std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
	doris::local_segment_path(file_dir, rowset_id, seg_id))},
	rowset_id, seg_id, doris::InvertedIndexStorageFormatPB::V2);
	auto st = index_file_writer->open(&index_meta);
	if (!st.has_value()) {
	std::cerr << "InvertedIndexFileWriter init error:" << st.error() << std::endl;
	return -1;
	}
	using T = std::decay_t<decltype(st)>;
	auto dir = std::forward<T>(st).value();
	auto analyzer = _CLNEW lucene::analysis::standard95::StandardAnalyzer();
	// auto analyzer = _CLNEW lucene::analysis::SimpleAnalyzer<char>();
	auto indexwriter = _CLNEW lucene::index::IndexWriter(dir, analyzer, true, true);
	indexwriter->setRAMBufferSizeMB(512);

source code from doris-thirdparty
https://github.com/apache/doris-thirdparty/blob/48fa9cc4ec32b40bf3b02338d0a1b2cdbc6408cf/src/core/CLucene/index/IndexWriter.h#L579

  /**
   * Constructs an IndexWriter for the index in <code>d</code>.
   * Text will be analyzed with <code>a</code>.  If <code>create</code>
   * is true, then a new, empty index will be created in
   * <code>d</code>, replacing the index already there, if any.
   *
   * @param d the index directory
   * @param a the analyzer to use
   * @param create <code>true</code> to create the index or overwrite
   *  the existing one; <code>false</code> to append to the existing
   *  index
   * @throws CorruptIndexException if the index is corrupt
   * @throws LockObtainFailedException if another writer
   *  has this index open (<code>write.lock</code> could not
   *  be obtained)
   * @throws IOException if the directory cannot be read/written to, or
   *  if it does not exist and <code>create</code> is
   *  <code>false</code> or if there is any other low-level
   *  IO error
   */
  explicit IndexWriter(CL_NS(store)::Directory* d, CL_NS(analysis)::Analyzer* a, const bool create, const bool closeDirOnShutdown=false);

I'm not familiar with c++, so I don't know exactly what caused the error. But I found some possible answers through copilot:

lucene::index::IndexWriter method is passed the first method of dir type may not be a std::shared_ptr<doris::segment_v2::DorisFSDirectory> 。

I'm not sure that's the case. But I got stuck trying to build 3.0.3-rc-04. I can't build properly.