From 0934b8fc91e3f08369d98fb9dcc24a1a5521a132 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 4 Jun 2025 22:30:07 +0100 Subject: [PATCH 1/4] test_analyze_wheel_abi: refactor test with explicit env Be explicit in the test-case matrix when to set env variable. This enables to check if environment variable is actually correctly reset. This commit passes unit tests. --- tests/integration/test_bundled_wheels.py | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index 046b8fc0..f7bc6296 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -1,13 +1,11 @@ from __future__ import annotations import importlib -import os import platform import sys import zipfile from argparse import Namespace from datetime import datetime, timezone -from os.path import isabs from pathlib import Path from unittest.mock import Mock @@ -23,69 +21,76 @@ @pytest.mark.parametrize( - ("file", "external_libs", "exclude"), + ("file", "external_libs", "exclude", "env"), [ ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libffi.so.5", "libpython2.7.so.1.0"}, frozenset(), + None, ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["libffi.so.5", "libpython2.7.so.1.0"]), + None, ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libffi.so.5", "libpython2.7.so.1.0"}, frozenset(["libffi.so.noexist", "libnoexist.so.*"]), + None, ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libpython2.7.so.1.0"}, frozenset(["libffi.so.[4,5]"]), + None, ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libffi.so.5", "libpython2.7.so.1.0"}, frozenset(["libffi.so.[6,7]"]), + None, ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libpython2.7.so.1.0"}, frozenset([f"{HERE}/*"]), + "LD_LIBRARY_PATH", ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libpython2.7.so.1.0"}, frozenset(["libffi.so.*"]), + None, ), - ("cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["*"])), + ("cffi-1.5.0-cp27-none-linux_x86_64.whl", set(), frozenset(["*"]), None), ( "python_snappy-0.5.2-pp260-pypy_41-linux_x86_64.whl", {"libsnappy.so.1"}, frozenset(), + None, ), ], ) -def test_analyze_wheel_abi(file, external_libs, exclude): +def test_analyze_wheel_abi(file, external_libs, exclude, env): # If exclude libs contain path, LD_LIBRARY_PATH need to be modified to find the libs # `lddtree.load_ld_paths` needs to be reloaded for it's `lru_cache`-ed. - modify_ld_library_path = any(isabs(e) for e in exclude) with pytest.MonkeyPatch.context() as cp: - if modify_ld_library_path: - cp.setenv("LD_LIBRARY_PATH", f"{HERE}") + if env: + cp.setenv(env, f"{HERE}") importlib.reload(lddtree) winfo = analyze_wheel_abi( Libc.GLIBC, Architecture.x86_64, HERE / file, exclude, False, True ) assert set(winfo.external_refs["manylinux_2_5_x86_64"].libs) == external_libs, ( - f"{HERE}, {exclude}, {os.environ}" + f"{HERE}, {exclude}, {env}" ) - if modify_ld_library_path: + if env: importlib.reload(lddtree) From 9f06c60d451a1b92bda33eaca25f882634b9d5e7 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 4 Jun 2025 22:39:22 +0100 Subject: [PATCH 2/4] test_analyze_wheel_abi: add failing test case Add a failing test case confiring that environment is not being correctly reset. --- tests/integration/test_bundled_wheels.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index f7bc6296..0c1d2227 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -59,6 +59,12 @@ frozenset([f"{HERE}/*"]), "LD_LIBRARY_PATH", ), + ( + "cffi-1.5.0-cp27-none-linux_x86_64.whl", + {"libffi.so.5", "libpython2.7.so.1.0"}, + frozenset([f"{HERE}/*"]), + None, + ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libpython2.7.so.1.0"}, From e2063e3893b01268c5821b47a663334e0ecc8c4e Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 4 Jun 2025 22:46:52 +0100 Subject: [PATCH 3/4] test_analyze_wheel_abi: reload and reimport wheel_abi It seems that in addition to reloading lddtree, reload of auditwheel.wheel_abi is needed to full flush and reset analyze_wheel_abi() lru_caches. --- tests/integration/test_bundled_wheels.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index 0c1d2227..7fa01700 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -11,6 +11,7 @@ import pytest +import auditwheel.wheel_abi from auditwheel import lddtree, main_repair from auditwheel.architecture import Architecture from auditwheel.libc import Libc @@ -87,7 +88,8 @@ def test_analyze_wheel_abi(file, external_libs, exclude, env): with pytest.MonkeyPatch.context() as cp: if env: cp.setenv(env, f"{HERE}") - importlib.reload(lddtree) + importlib.reload(lddtree) + importlib.reload(auditwheel.wheel_abi) winfo = analyze_wheel_abi( Libc.GLIBC, Architecture.x86_64, HERE / file, exclude, False, True @@ -96,8 +98,8 @@ def test_analyze_wheel_abi(file, external_libs, exclude, env): f"{HERE}, {exclude}, {env}" ) - if env: - importlib.reload(lddtree) + importlib.reload(lddtree) + importlib.reload(auditwheel.wheel_abi) def test_analyze_wheel_abi_pyfpe(): From 496e92ef4e00cbb34e1b18394f0d993eb233b764 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Wed, 16 Apr 2025 12:36:33 +0100 Subject: [PATCH 4/4] lddtree: support alternative LD_LIBRARY_PATH called AUDITWHEEL_LD_LIBRARY_PATH Sometimes wheels are different only just by shared libraries that are vendored in. Sometimes the wheel being repaired and the libraries that need to be vendored in are accessible on the host that was not used for building the wheel. In such cases, it could be the case that shared libraries used by python that is executing the auditwheel script, are incompatible (too old or too new) than those that need to be vendored. In such cases, LD_LIBRARY_PATH is not convenient to use, as the python interpreter for the auditwheel script is crashing. Add an AUDITWHEEL_LD_LIBRARY_PATH whichis used by lddtree, but otherwise does not affect the python interpreter executing auditwheel script. This helps vendoring in older libraries from an alternative path, into a wheel built against an older python, whilst otherwise running auditwheel repair on a modern system with a much newer python. This is particularly useful when stripping a wheel of vendored libraries, and repair it again to update the shared library dependencies coming from an unpacked chroot. Separately it also helps creating alternative wheels with shared libraries rebuilt with higher march settings. Given that python upstream doesn't support loading optimised libraries as has been implemented and supported in Intel ClearLinux Python which allows one to have alternative march setting shared library deps. --- src/auditwheel/lddtree.py | 13 +++++++++++-- tests/integration/test_bundled_wheels.py | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/auditwheel/lddtree.py b/src/auditwheel/lddtree.py index 44785791..661446a4 100644 --- a/src/auditwheel/lddtree.py +++ b/src/auditwheel/lddtree.py @@ -321,8 +321,17 @@ def load_ld_paths( """ ldpaths: dict[str, list[str]] = {"conf": [], "env": [], "interp": []} - # Load up $LD_LIBRARY_PATH. - env_ldpath = os.environ.get("LD_LIBRARY_PATH") + # Load up $AUDITWHEEL_LD_LIBRARY_PATH and $LD_LIBRARY_PATH + env_ldpath = ":".join( + filter( + None, + ( + os.environ.get("AUDITWHEEL_LD_LIBRARY_PATH"), + os.environ.get("LD_LIBRARY_PATH"), + ), + ) + ) + if env_ldpath is not None: if root != "/": log.warning("ignoring LD_LIBRARY_PATH due to ROOT usage") diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index 7fa01700..97f12487 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -60,6 +60,12 @@ frozenset([f"{HERE}/*"]), "LD_LIBRARY_PATH", ), + ( + "cffi-1.5.0-cp27-none-linux_x86_64.whl", + {"libpython2.7.so.1.0"}, + frozenset([f"{HERE}/*"]), + "AUDITWHEEL_LD_LIBRARY_PATH", + ), ( "cffi-1.5.0-cp27-none-linux_x86_64.whl", {"libffi.so.5", "libpython2.7.so.1.0"},