From 7a132713989a5df5797ccb4c5fdee4525fbb464c Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Sat, 2 Sep 2023 23:54:21 -0500 Subject: [PATCH 1/4] Repair ELF executables in the "scripts" directory --- src/auditwheel/repair.py | 60 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index c172b471..987e50ae 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -66,7 +66,7 @@ def repair_wheel( if not exists(dest_dir): os.mkdir(dest_dir) - # here, fn is a path to a python extension library in + # here, fn is a path to an ELF file (lib or executable) in # the wheel, and v['libs'] contains its required libs for fn, v in external_refs_by_fn.items(): ext_libs: dict[str, str] = v[abis[0]]["libs"] @@ -92,6 +92,9 @@ def repair_wheel( patcher.replace_needed(fn, *replacements) if len(ext_libs) > 0: + if _path_is_script(fn): + fn = _replace_elf_script_with_shim(match.group("name"), fn) + new_rpath = os.path.relpath(dest_dir, os.path.dirname(fn)) new_rpath = os.path.join("$ORIGIN", new_rpath) append_rpath_within_wheel(fn, new_rpath, ctx.name, patcher) @@ -232,3 +235,58 @@ def _resolve_rpath_tokens(rpath: str, lib_base_dir: str) -> str: rpath = rpath.replace(f"${token}", target) # $TOKEN rpath = rpath.replace(f"${{{token}}}", target) # ${TOKEN} return rpath + + +def _path_is_script(path: str) -> bool: + # Looks something like "uWSGI-2.0.21.data/scripts/uwsgi" + components = path.split("/") + return ( + len(components) == 3 + and components[0].endswith(".data") + and components[1] == "scripts" + ) + + +def _replace_elf_script_with_shim(package_name: str, orig_path: str) -> str: + """Move an ELF script and replace it with a shim. + + We can't directly rewrite the RPATH of ELF executables in the "scripts" + directory since scripts aren't installed to a consistent relative path to + platlib files. + + Instead, we move the executable into a special directory in platlib and put + a shim script in its place which execs the real executable. + + More context: https://github.com/pypa/auditwheel/issues/340 + + Returns the new path of the moved executable. + """ + scripts_dir = f"{package_name}.scripts" + os.makedirs(scripts_dir, exist_ok=True) + + new_path = os.path.join(scripts_dir, os.path.basename(orig_path)) + os.rename(orig_path, new_path) + + with open(orig_path, "w") as f: + f.write(_script_shim(new_path)) + os.chmod(orig_path, os.stat(new_path).st_mode) + + return new_path + + +def _script_shim(binary_path: str) -> str: + return """\ +#!python +import os +import sys +import sysconfig + + +if __name__ == "__main__": + os.execv( + os.path.join(sysconfig.get_path("platlib"), {binary_path!r}), + sys.argv, + ) +""".format( + binary_path=binary_path, + ) From 051e3d0020aa07ca67dee0a197cf0cf490d159df Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Thu, 4 Jan 2024 02:42:25 -0600 Subject: [PATCH 2/4] Make integration tests work under podman podman was failing with this error due to (apparently) lacking support for abbreviated sha256 references: docker.errors.APIError: 500 Server Error for http+docker://localhost/v1.41/containers/create: Internal Server Error ("normalizing image: normalizing name for compat API: sha256:a455ef9d0843: invalid format: no 64-byte hexadecimal value") Using the full SHA256 hash works under podman (and presumably works under Docker too?). --- tests/integration/test_manylinux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 39b33dc4..08a05140 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -175,7 +175,7 @@ def tmp_docker_image(base, commands, setup_env={}): logger.info("Made image %s based on %s", image.short_id, base) try: - yield image.short_id + yield image.id finally: client = image.client client.images.remove(image.id) From ac46754b300636345eb5b00d8fee566295f338ed Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Thu, 4 Jan 2024 03:22:31 -0600 Subject: [PATCH 3/4] Add test for ELF binaries in scripts directory --- .gitignore | 3 ++- tests/integration/test_manylinux.py | 22 +++++++++++++++++++ tests/integration/testpackage/setup.py | 14 +++++++++--- .../testpackage/testprogram_nodeps.c | 19 ++++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 tests/integration/testpackage/testpackage/testprogram_nodeps.c diff --git a/.gitignore b/.gitignore index 3ab26439..92651bf5 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,5 @@ target/ # Generated by test script *.zip wheelhoust-* -tests/testpackage/testpackage/testprogram +tests/integration/testpackage/testpackage/testprogram +tests/integration/testpackage/testpackage/testprogram_nodeps diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 08a05140..45a73875 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -404,6 +404,28 @@ def test_build_wheel_with_binary_executable( ) assert output.strip() == "2.25" + # Both testprogram and testprogram_nodeps square a number, but: + # * testprogram links against libgsl and had to have its RPATH + # rewritten. + # * testprogram_nodeps links against no shared libraries and wasn't + # rewritten. + # + # Both executables should work when called from the installed bin directory. + assert docker_exec(docker_python, ["/usr/local/bin/testprogram", "4"]) == "16\n" + assert docker_exec(docker_python, ["/usr/local/bin/testprogram_nodeps", "4"]) == "16\n" + + # testprogram should be a Python shim since we had to rewrite its RPATH. + assert ( + docker_exec(docker_python, ["head", "-n1", "/usr/local/bin/testprogram"]) + == "#!/usr/local/bin/python\n" + ) + + # testprogram_nodeps should be the unmodified ELF binary. + assert ( + docker_exec(docker_python, ["head", "-c4", "/usr/local/bin/testprogram_nodeps"]) + == "\x7fELF" + ) + def test_build_repair_pure_wheel(self, any_manylinux_container, io_folder): policy, tag, manylinux_ctr = any_manylinux_container diff --git a/tests/integration/testpackage/setup.py b/tests/integration/testpackage/setup.py index 8fda5a3c..1702d598 100644 --- a/tests/integration/testpackage/setup.py +++ b/tests/integration/testpackage/setup.py @@ -2,14 +2,22 @@ import subprocess + from setuptools import setup -cmd = "gcc testpackage/testprogram.c -lgsl -lgslcblas -o testpackage/testprogram" -subprocess.check_call(cmd.split()) +subprocess.check_call(("gcc", "testpackage/testprogram.c", "-lgsl", "-lgslcblas", "-o", "testpackage/testprogram")) +subprocess.check_call(("gcc", "testpackage/testprogram_nodeps.c", "-o", "testpackage/testprogram_nodeps")) setup( name="testpackage", version="0.0.1", packages=["testpackage"], - package_data={"testpackage": ["testprogram"]}, + package_data={"testpackage": ["testprogram", "testprogram_nodeps"]}, + # This places these files at a path like + # "testpackage-0.0.1.data/scripts/testprogram", which is needed to test + # rewriting ELF binaries installed into the scripts directory. + # + # Note that using scripts=[] doesn't work here since setuptools expects the + # scripts to be text and tries to decode them using UTF-8. + data_files=[("../scripts", ["testpackage/testprogram", "testpackage/testprogram_nodeps"])], ) diff --git a/tests/integration/testpackage/testpackage/testprogram_nodeps.c b/tests/integration/testpackage/testpackage/testprogram_nodeps.c new file mode 100644 index 00000000..ec65595c --- /dev/null +++ b/tests/integration/testpackage/testpackage/testprogram_nodeps.c @@ -0,0 +1,19 @@ +/* A simple example program to square a number using no shared libraries. */ + +#include +#include + +int main(int argc, char **argv) +{ + int x; + + if (argc != 2) + { + fputs("Expected exactly one command line argument\n", stderr); + return EXIT_FAILURE; + } + + x = atoi(argv[1]); + printf("%d\n", x*x); + return EXIT_SUCCESS; +} From c52a868eb45359ca2c4202c6ac43fa8825508522 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:23:37 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/integration/test_manylinux.py | 9 +++++++-- tests/integration/testpackage/setup.py | 20 ++++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index 45a73875..09d1c9a9 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -412,7 +412,10 @@ def test_build_wheel_with_binary_executable( # # Both executables should work when called from the installed bin directory. assert docker_exec(docker_python, ["/usr/local/bin/testprogram", "4"]) == "16\n" - assert docker_exec(docker_python, ["/usr/local/bin/testprogram_nodeps", "4"]) == "16\n" + assert ( + docker_exec(docker_python, ["/usr/local/bin/testprogram_nodeps", "4"]) + == "16\n" + ) # testprogram should be a Python shim since we had to rewrite its RPATH. assert ( @@ -422,7 +425,9 @@ def test_build_wheel_with_binary_executable( # testprogram_nodeps should be the unmodified ELF binary. assert ( - docker_exec(docker_python, ["head", "-c4", "/usr/local/bin/testprogram_nodeps"]) + docker_exec( + docker_python, ["head", "-c4", "/usr/local/bin/testprogram_nodeps"] + ) == "\x7fELF" ) diff --git a/tests/integration/testpackage/setup.py b/tests/integration/testpackage/setup.py index 1702d598..a3e90d82 100644 --- a/tests/integration/testpackage/setup.py +++ b/tests/integration/testpackage/setup.py @@ -2,11 +2,21 @@ import subprocess - from setuptools import setup -subprocess.check_call(("gcc", "testpackage/testprogram.c", "-lgsl", "-lgslcblas", "-o", "testpackage/testprogram")) -subprocess.check_call(("gcc", "testpackage/testprogram_nodeps.c", "-o", "testpackage/testprogram_nodeps")) +subprocess.check_call( + ( + "gcc", + "testpackage/testprogram.c", + "-lgsl", + "-lgslcblas", + "-o", + "testpackage/testprogram", + ) +) +subprocess.check_call( + ("gcc", "testpackage/testprogram_nodeps.c", "-o", "testpackage/testprogram_nodeps") +) setup( name="testpackage", @@ -19,5 +29,7 @@ # # Note that using scripts=[] doesn't work here since setuptools expects the # scripts to be text and tries to decode them using UTF-8. - data_files=[("../scripts", ["testpackage/testprogram", "testpackage/testprogram_nodeps"])], + data_files=[ + ("../scripts", ["testpackage/testprogram", "testpackage/testprogram_nodeps"]) + ], )