diff --git a/.gitlab/prepare-oci-package.sh b/.gitlab/prepare-oci-package.sh index 4f8a3ec13f0..4f7579e9998 100755 --- a/.gitlab/prepare-oci-package.sh +++ b/.gitlab/prepare-oci-package.sh @@ -27,10 +27,4 @@ echo -n "$PYTHON_PACKAGE_VERSION" > sources/version cp -r ../pywheels-dep/site-packages* sources/ddtrace_pkgs -cp ../lib-injection/sitecustomize.py sources/ -cp ../min_compatible_versions.csv sources/ -cp ../lib-injection/telemetry-forwarder.sh sources/ - -clean-apt install python3 -echo "Deduplicating package files" -python3 ../lib-injection/dedupe.py sources/ddtrace_pkgs/ +cp ../lib-injection/sources/* sources/ diff --git a/lib-injection/dedupe.py b/lib-injection/dedupe.py deleted file mode 100644 index 405344550b4..00000000000 --- a/lib-injection/dedupe.py +++ /dev/null @@ -1,57 +0,0 @@ -import collections -import glob -import os -from pathlib import Path -import shutil -import subprocess -import sys - - -if len(sys.argv) != 2: - print("Usage: python dedupe.py ") - print("Example: python dedupe.py sources/ddtrace_pkgs") - sys.exit(1) - -source_dir = Path(sys.argv[1]) - -# Find all duplicate files -print("Finding duplicate files") -file_hashes = collections.defaultdict(set) -for src in glob.glob(f"{source_dir}/**/*", recursive=True): - if not os.path.isfile(src): - continue - res = subprocess.check_output(["sha256sum", str(src)]) - file_hash, _, _ = res.decode().partition(" ") - file_hashes[file_hash].add(Path(src)) - - -# Replace shared files with soft links -shared_dir = source_dir / "shared" -try: - shutil.rmtree(shared_dir) -except Exception: - pass -os.makedirs(shared_dir) - -for file_hash in file_hashes: - # Skip unique files that aren't duplicates - if len(file_hashes[file_hash]) <= 1: - continue - - # Copy the first file to the shared directory with the name shared/_ - src = next(iter(file_hashes[file_hash])) - basename = os.path.basename(src) - dest = shared_dir / f"{file_hash}_{basename}" - print(f"Copying {src} to {dest}") - shutil.copy(src, dest) - - for src in file_hashes[file_hash]: - # Replace the duplicate file with a symlink to the shared file - dest_rel = dest.relative_to(source_dir) - # Convert the path to the src file with a relative path to the shared file - # e.g. `site-packages-ddtrace-py3.11-manylinux2014/xmltodict.py` -> `../shared/_xmltodict.py` - # DEV: This means we don't need to know the absolute path of where the files end up on disk - src_rel = Path(*([".." for _ in src.relative_to(source_dir).parts[:-1]] + [dest_rel])) - print(f"Replacing {src} with symlink to {src_rel}") - os.remove(src) - os.symlink(src_rel, src) diff --git a/lib-injection/denied_executables.txt b/lib-injection/sources/denied_executables.txt similarity index 100% rename from lib-injection/denied_executables.txt rename to lib-injection/sources/denied_executables.txt diff --git a/lib-injection/min_compatible_versions.csv b/lib-injection/sources/min_compatible_versions.csv similarity index 100% rename from lib-injection/min_compatible_versions.csv rename to lib-injection/sources/min_compatible_versions.csv diff --git a/lib-injection/sitecustomize.py b/lib-injection/sources/sitecustomize.py similarity index 93% rename from lib-injection/sitecustomize.py rename to lib-injection/sources/sitecustomize.py index 206f79d1242..c41967b03b5 100644 --- a/lib-injection/sitecustomize.py +++ b/lib-injection/sources/sitecustomize.py @@ -26,6 +26,7 @@ def parse_version(version: str) -> Tuple: return Version(parsed_version, constraint) +SCRIPT_DIR = os.path.dirname(__file__) RUNTIMES_ALLOW_LIST = { "cpython": {"min": parse_version("3.7"), "max": parse_version("3.13")}, } @@ -39,8 +40,11 @@ def parse_version(version: str) -> Tuple: PYTHON_RUNTIME = None PKGS_ALLOW_LIST = None EXECUTABLES_DENY_LIST = None -VERSION_COMPAT_FILE_LOCATIONS = ("../datadog-lib/min_compatible_versions.csv", "min_compatible_versions.csv") -EXECUTABLE_DENY_LOCATION = "denied_executables.txt" +VERSION_COMPAT_FILE_LOCATIONS = ( + os.path.abspath(os.path.join(SCRIPT_DIR, "../datadog-lib/min_compatible_versions.csv")), + os.path.abspath(os.path.join(SCRIPT_DIR, "min_compatible_versions.csv")), +) +EXECUTABLE_DENY_LOCATION = os.path.abspath(os.path.join(SCRIPT_DIR, "denied_executables.txt")) def build_installed_pkgs(): @@ -80,12 +84,15 @@ def build_min_pkgs(): def build_denied_executables(): denied_executables = set() + _log("Checking denied-executables list", level="debug") if os.path.exists(EXECUTABLE_DENY_LOCATION): with open(EXECUTABLE_DENY_LOCATION, "r") as denyfile: + _log("Found deny-list file", level="debug") for line in denyfile.readlines(): cleaned = line.strip("\n") denied_executables.add(cleaned) denied_executables.add(os.path.basename(cleaned)) + _log(f"Built denied-executables list of {len(denied_executables)} entries", level="debug") return denied_executables @@ -173,10 +180,13 @@ def package_is_compatible(package_name, package_version): def get_first_incompatible_sysarg(): + _log(f"Checking sysargs: len(argv): {len(sys.argv)}", level="debug") if len(sys.argv) <= 1: return argument = sys.argv[0] + _log(f"Is argument {argument} in deny-list?", level="debug") if argument in EXECUTABLES_DENY_LIST or os.path.basename(argument) in EXECUTABLES_DENY_LIST: + _log(f"argument {argument} is in deny-list", level="debug") return argument @@ -203,8 +213,7 @@ def _inject(): current_platform = "manylinux2014" if _get_clib() == "gnu" else "musllinux_1_1" _log("detected platform %s" % current_platform, level="debug") - script_dir = os.path.dirname(__file__) - pkgs_path = os.path.join(script_dir, "ddtrace_pkgs") + pkgs_path = os.path.join(SCRIPT_DIR, "ddtrace_pkgs") _log("ddtrace_pkgs path is %r" % pkgs_path, level="debug") _log("ddtrace_pkgs contents: %r" % os.listdir(pkgs_path), level="debug") @@ -308,8 +317,8 @@ def _inject(): # - Add the custom site-packages directory to PYTHONPATH to ensure the ddtrace package can be loaded # - Add the ddtrace bootstrap dir to the PYTHONPATH to achieve the same effect as ddtrace-run. python_path = os.getenv("PYTHONPATH", "").split(os.pathsep) - if script_dir in python_path: - python_path.remove(script_dir) + if SCRIPT_DIR in python_path: + python_path.remove(SCRIPT_DIR) python_path.insert(-1, site_pkgs_path) bootstrap_dir = os.path.abspath(os.path.dirname(ddtrace.bootstrap.sitecustomize.__file__)) python_path.insert(0, bootstrap_dir) diff --git a/lib-injection/telemetry-forwarder.sh b/lib-injection/sources/telemetry-forwarder.sh similarity index 100% rename from lib-injection/telemetry-forwarder.sh rename to lib-injection/sources/telemetry-forwarder.sh diff --git a/releasenotes/notes/fix-oci-denylist-080592ca52e45681.yaml b/releasenotes/notes/fix-oci-denylist-080592ca52e45681.yaml new file mode 100644 index 00000000000..3656dddd050 --- /dev/null +++ b/releasenotes/notes/fix-oci-denylist-080592ca52e45681.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + SSI: This fix ensures injection denylist is included in published OCI package.