do not support bundled 3rd party code any more, fixes borgbackup#6316

Author: ThomasWaldmann

docs/installation.rst

@@ -159,20 +159,18 @@ Dependencies
 To install Borg from a source package (including pip), you have to install the
 following dependencies first:
 
-* `Python 3`_ >= 3.8.0, plus development headers.
-* OpenSSL_ >= 1.1.1, plus development headers.
-* libacl_ (which depends on libattr_), both plus development headers.
-* We have bundled code of the following packages, but borg by default (see
-  setup.py if you want to change that) prefers a shared library if it can
-  be found on the system (lib + dev headers) at build time:
+* `Python 3`_ >= 3.8.0 (interpreter plus development headers)
+* Libraries (library plus development headers):
 
+  - OpenSSL_ >= 1.1.1
+  - libacl_ (which depends on libattr_)
   - liblz4_ >= 1.7.0 (r129)
   - libzstd_ >= 1.3.0
   - libxxhash >= 0.8.1 (0.8.0 might work also)
 * pkg-config (cli tool) and pkgconfig python package (borg uses these to
   discover header and library location - if it can't import pkgconfig and
   is not pointed to header/library locations via env vars [see setup.py],
-  it will fall back to using the bundled code, see above).
+  it will raise a fatal error).
   **These must be present before invoking setup.py!**
 * some other Python dependencies, pip will automatically install them for you.
 * optionally, if you wish to mount an archive as a FUSE filesystem, you need

setup.py

@@ -26,35 +26,25 @@
 
 is_win32 = sys.platform.startswith('win32')
 
-# How the build process finds the system libs / uses the bundled code:
+# How the build process finds the system libs:
 #
-# 1. it will try to use (system) libs (see 1.1. and 1.2.),
-#    except if you use these env vars to force using the bundled code:
-#    BORG_USE_BUNDLED_XXX undefined  -->  try using system lib
-#    BORG_USE_BUNDLED_XXX=YES        -->  use the bundled code
-#    Note: do not use =NO, that is not supported!
-# 1.1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there.
-# 1.2. if not and pkg-config can locate the lib, the lib located by
-#      pkg-config will be used. We use the pkg-config tool via the pkgconfig
-#      python package, which must be installed before invoking setup.py.
-#      if pkgconfig is not installed, this step is skipped.
-# 2. if no system lib could be located via 1.1. or 1.2., it will fall back
-#    to using the bundled code.
-
-# OpenSSL is required as a (system) lib in any case as we do not bundle it.
-# Thus, only step 1.1. and 1.2. apply to openssl (but not 1. and 2.).
-# needed: openssl >=1.0.2 or >=1.1.0 (or compatible)
+# 1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there.
+# 2. if not and pkg-config can locate the lib, the lib located by
+#    pkg-config will be used. We use the pkg-config tool via the pkgconfig
+#    python package, which must be installed before invoking setup.py.
+#    if pkgconfig is not installed, this step is skipped.
+# 3. otherwise raise a fatal error.
+
+# needed: >=1.1.1 (or compatible)
 system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX')
 
 # needed: lz4 (>= 1.7.0 / r129)
-prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4'))
 system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX')
 
 # needed: zstd (>= 1.3.0)
-prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD'))
 system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
 
-prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH'))
+# needed: xxhash (>= 0.8.1)
 system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX')
 
 # Number of threads to use for cythonize, not used on windows
@@ -187,14 +177,13 @@ def members_appended(*ds):
 
     compress_ext_kwargs = members_appended(
         dict(sources=[compress_source]),
-        setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4, system_prefix_liblz4),
-        setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, system_prefix_libzstd,
-                                       multithreaded=False, legacy=False),
+        setup_compress.lz4_ext_kwargs(pc, system_prefix_liblz4),
+        setup_compress.zstd_ext_kwargs(pc, system_prefix_libzstd),
     )
 
     checksums_ext_kwargs = members_appended(
         dict(sources=[checksums_source]),
-        setup_checksums.xxhash_ext_kwargs(pc, prefer_system_libxxhash, system_prefix_libxxhash),
+        setup_checksums.xxhash_ext_kwargs(pc, system_prefix_libxxhash),
     )
 
     ext_modules += [

setup_checksums.py

@@ -3,40 +3,15 @@
 import os
 
 
-def multi_join(paths, *path_segments):
-    """apply os.path.join on a list of paths"""
-    return [os.path.join(*(path_segments + (path,))) for path in paths]
-
-
-# xxhash files, structure as seen in xxhash project repository:
-
-# path relative (to this file) to the bundled library source code files
-xxhash_bundled_path = 'src/borg/algorithms/xxh64'
-
-xxhash_sources = [
-    'xxhash.c',
-]
-
-xxhash_includes = [
-    '',
-]
-
-
-def xxhash_ext_kwargs(pc, prefer_system, system_prefix):
-    if prefer_system:
-        if system_prefix:
-            print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]')
-            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                        library_dirs=[os.path.join(system_prefix, 'lib')],
-                        libraries=['xxhash'])
-
-        if pc and pc.installed('libxxhash', '>= 0.7.3'):
-            print('Detected and preferring libxxhash [via pkg-config]')
-            return pc.parse('libxxhash')
-
-    print('Using bundled xxhash')
-    sources = multi_join(xxhash_sources, xxhash_bundled_path)
-    include_dirs = multi_join(xxhash_includes, xxhash_bundled_path)
-    define_macros = [('BORG_USE_BUNDLED_XXHASH', 'YES')]
-    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
-
+def xxhash_ext_kwargs(pc, system_prefix):
+    if system_prefix:
+        print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]')
+        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                    library_dirs=[os.path.join(system_prefix, 'lib')],
+                    libraries=['xxhash'])
+
+    if pc and pc.installed('libxxhash', '>= 0.7.3'):
+        print('Detected and preferring libxxhash [via pkg-config]')
+        return pc.parse('libxxhash')
+
+    raise Exception('Could not find xxhash lib/headers, please set BORG_LIBXXHASH_PREFIX')

setup_compress.py

@@ -3,133 +3,29 @@
 import os
 
 
-def multi_join(paths, *path_segments):
-    """apply os.path.join on a list of paths"""
-    return [os.path.join(*(path_segments + (path,))) for path in paths]
+def zstd_ext_kwargs(pc, system_prefix):
+    if system_prefix:
+        print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
+        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                    library_dirs=[os.path.join(system_prefix, 'lib')],
+                    libraries=['zstd'])
 
+    if pc and pc.installed('libzstd', '>= 1.3.0'):
+        print('Detected and preferring libzstd [via pkg-config]')
+        return pc.parse('libzstd')
 
-# zstd files, structure as seen in zstd project repository:
+    raise Exception('Could not find zstd lib/headers, please set BORG_LIBZSTD_PREFIX')
 
-# path relative (to this file) to the bundled library source code files
-zstd_bundled_path = 'src/borg/algorithms/zstd'
 
-zstd_sources = [
-    'lib/common/debug.c',
-    'lib/common/entropy_common.c',
-    'lib/common/error_private.c',
-    'lib/common/fse_decompress.c',
-    'lib/common/pool.c',
-    'lib/common/threading.c',
-    'lib/common/xxhash.c',
-    'lib/common/zstd_common.c',
-    'lib/compress/fse_compress.c',
-    'lib/compress/hist.c',
-    'lib/compress/huf_compress.c',
-    'lib/compress/zstd_compress.c',
-    'lib/compress/zstd_compress_literals.c',
-    'lib/compress/zstd_compress_sequences.c',
-    'lib/compress/zstd_compress_superblock.c',
-    'lib/compress/zstd_double_fast.c',
-    'lib/compress/zstd_fast.c',
-    'lib/compress/zstd_lazy.c',
-    'lib/compress/zstd_ldm.c',
-    'lib/compress/zstd_opt.c',
-    'lib/compress/zstdmt_compress.c',
-    'lib/decompress/huf_decompress.c',
-    'lib/decompress/zstd_ddict.c',
-    'lib/decompress/zstd_decompress.c',
-    'lib/decompress/zstd_decompress_block.c',
-    'lib/dictBuilder/cover.c',
-    'lib/dictBuilder/divsufsort.c',
-    'lib/dictBuilder/fastcover.c',
-    'lib/dictBuilder/zdict.c',
-]
+def lz4_ext_kwargs(pc, system_prefix):
+    if system_prefix:
+        print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
+        return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                    library_dirs=[os.path.join(system_prefix, 'lib')],
+                    libraries=['lz4'])
 
-zstd_sources_legacy = [
-    'lib/deprecated/zbuff_common.c',
-    'lib/deprecated/zbuff_compress.c',
-    'lib/deprecated/zbuff_decompress.c',
-    'lib/legacy/zstd_v01.c',
-    'lib/legacy/zstd_v02.c',
-    'lib/legacy/zstd_v03.c',
-    'lib/legacy/zstd_v04.c',
-    'lib/legacy/zstd_v05.c',
-    'lib/legacy/zstd_v06.c',
-    'lib/legacy/zstd_v07.c',
-]
+    if pc and pc.installed('liblz4', '>= 1.7.0'):
+        print('Detected and preferring liblz4 [via pkg-config]')
+        return pc.parse('liblz4')
 
-zstd_includes = [
-    'lib',
-    'lib/common',
-    'lib/compress',
-    'lib/decompress',
-    'lib/dictBuilder',
-]
-
-zstd_includes_legacy = [
-    'lib/deprecated',
-    'lib/legacy',
-]
-
-
-def zstd_ext_kwargs(pc, prefer_system, system_prefix, multithreaded=False, legacy=False):
-    if prefer_system:
-        if system_prefix:
-            print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
-            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                        library_dirs=[os.path.join(system_prefix, 'lib')],
-                        libraries=['zstd'])
-
-        if pc and pc.installed('libzstd', '>= 1.3.0'):
-            print('Detected and preferring libzstd [via pkg-config]')
-            return pc.parse('libzstd')
-
-    print('Using bundled ZSTD')
-    sources = multi_join(zstd_sources, zstd_bundled_path)
-    if legacy:
-        sources += multi_join(zstd_sources_legacy, zstd_bundled_path)
-    include_dirs = multi_join(zstd_includes, zstd_bundled_path)
-    if legacy:
-        include_dirs += multi_join(zstd_includes_legacy, zstd_bundled_path)
-    extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
-    # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
-    if legacy:
-        extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
-    if multithreaded:
-        extra_compile_args += ['-DZSTD_MULTITHREAD', ]
-    define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')]
-    return dict(sources=sources, include_dirs=include_dirs,
-                extra_compile_args=extra_compile_args, define_macros=define_macros)
-
-
-# lz4 files, structure as seen in lz4 project repository:
-
-# path relative (to this file) to the bundled library source code files
-lz4_bundled_path = 'src/borg/algorithms/lz4'
-
-lz4_sources = [
-    'lib/lz4.c',
-]
-
-lz4_includes = [
-    'lib',
-]
-
-
-def lz4_ext_kwargs(pc, prefer_system, system_prefix):
-    if prefer_system:
-        if system_prefix:
-            print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
-            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                        library_dirs=[os.path.join(system_prefix, 'lib')],
-                        libraries=['lz4'])
-
-        if pc and pc.installed('liblz4', '>= 1.7.0'):
-            print('Detected and preferring liblz4 [via pkg-config]')
-            return pc.parse('liblz4')
-
-    print('Using bundled LZ4')
-    sources = multi_join(lz4_sources, lz4_bundled_path)
-    include_dirs = multi_join(lz4_includes, lz4_bundled_path)
-    define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')]
-    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
+    raise Exception('Could not find lz4 lib/headers, please set BORG_LIBLZ4_PREFIX')

src/borg/algorithms/checksums.pyx

@@ -12,7 +12,7 @@ cdef extern from "crc32_dispatch.c":
     int _have_clmul "have_clmul"()
 
 
-cdef extern from "xxhash-libselect.h":
+cdef extern from "xxhash.h":
     ctypedef struct XXH64_canonical_t:
         char digest[8]
 

src/borg/algorithms/lz4-libselect.h

@@ -30,13 +30,13 @@ from .helpers import Buffer, DecompressionError
 
 API_VERSION = '1.2_02'
 
-cdef extern from "algorithms/lz4-libselect.h":
+cdef extern from "lz4.h":
     int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
     int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
     int LZ4_compressBound(int inputSize) nogil
 
 
-cdef extern from "algorithms/zstd-libselect.h":
+cdef extern from "zstd.h":
     size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int  compressionLevel) nogil
     size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize) nogil
     size_t ZSTD_compressBound(size_t srcSize) nogil