From dc30729b3bf9dc3d268408eec42831322d755032 Mon Sep 17 00:00:00 2001 From: vince62s Date: Fri, 9 Jan 2026 17:26:59 +0100 Subject: [PATCH 1/4] new try --- .github/workflows/ci.yml | 34 ++++++++++--------- bindings/python/setup.py | 24 +++++-------- .../tools/prepare_build_environment_macos.sh | 9 ++--- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de99f65..91a6284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,12 +121,14 @@ jobs: output-dir: wheelhouse env: # ---- Build selection ---- + # Linux/Windows: Python 3.10, 3.11, 3.12 + # macOS: Python 3.10, 3.11 only (skip 3.12) CIBW_BUILD: "cp310-* cp311-* cp312-*" - CIBW_SKIP: "pp* *-musllinux_*" + CIBW_SKIP: "pp* *-musllinux_* cp312-macosx*" CIBW_ARCHS: ${{ matrix.arch }} # ---- Dependencies ---- - CIBW_BEFORE_BUILD: pip install pybind11>=2.10.4 + CIBW_BEFORE_BUILD: pip install pybind11==2.10.1 # ---- Linux ---- CIBW_BEFORE_ALL_LINUX: bindings/python/tools/prepare_build_environment_linux.sh @@ -138,24 +140,24 @@ jobs: # ---- macOS ---- CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh + CIBW_BEFORE_BUILD_MACOS: | + # Copy ICU dylibs into source tree before build + export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" + mkdir -p {project}/pyonmttok/icu/lib + if [ -d "$ICU_ROOT/lib" ] && compgen -G "$ICU_ROOT/lib/*.dylib" > /dev/null; then + echo "Copying ICU dylibs to source tree" + cp -a "$ICU_ROOT/lib/"*.dylib {project}/pyonmttok/icu/lib/ + ls -lh {project}/pyonmttok/icu/lib/ + else + echo "ERROR: No ICU dylibs found in $ICU_ROOT/lib" + exit 1 + fi CIBW_ENVIRONMENT_MACOS: | TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install ICU_ROOT=${GITHUB_WORKSPACE}/icu DYLD_LIBRARY_PATH=${GITHUB_WORKSPACE}/icu/lib:${DYLD_LIBRARY_PATH} - CIBW_REPAIR_WHEEL_COMMAND_MACOS: | - set -e - echo "=== Bundling ICU into wheel ===" - delocate-wheel -v -w {dest_dir} {wheel} - - REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) - echo "=== Inspecting repaired wheel ===" - unzip -q "$REPAIRED_WHEEL" -d /tmp/wheel_check - SO_FILE=$(find /tmp/wheel_check -name "_ext*.so" | head -n1) - otool -L "$SO_FILE" - rm -rf /tmp/wheel_check - echo "=== Wheel ready: $REPAIRED_WHEEL ===" - - + # No repair step for macOS + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh diff --git a/bindings/python/setup.py b/bindings/python/setup.py index dc786f1..232800f 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,21 +1,17 @@ import os import sys - import pybind11 - from setuptools import Extension, find_packages, setup include_dirs = [pybind11.get_include()] library_dirs = [] libraries = ["OpenNMTTokenizer"] - def _get_long_description(): readme_path = "README.md" with open(readme_path, encoding="utf-8") as readme_file: return readme_file.read() - def _get_project_version(): base_dir = os.path.dirname(os.path.abspath(__file__)) version_path = os.path.join(base_dir, "pyonmttok", "version.py") @@ -24,7 +20,6 @@ def _get_project_version(): exec(fp.read(), version) return version["__version__"] - def _maybe_add_library_root(lib_name, header_only=False): root = os.environ.get(f"{lib_name}_ROOT") if root is None: @@ -38,7 +33,6 @@ def _maybe_add_library_root(lib_name, header_only=False): return lib_dir return None - _maybe_add_library_root("TOKENIZER") icu_lib_dir = _maybe_add_library_root("ICU") @@ -49,18 +43,16 @@ def _maybe_add_library_root(lib_name, header_only=False): if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") if icu_lib_dir: + # Add ICU libraries to the libraries list for linking icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] libraries.extend(icu_libs) - - # Link ICU with full paths so delocate can see it - for lib in icu_libs: - full_lib_path = os.path.join(icu_lib_dir, f"lib{lib}.dylib") - if os.path.isfile(full_lib_path): - ldflags.append(full_lib_path) - - # rpath for runtime - ldflags.append("-Wl,-rpath,@loader_path/../icu/lib") - + # Set rpath to look for ICU dylibs in the icu/lib directory within the package + ldflags.extend([ + "-Wl,-rpath,@loader_path/icu/lib", + "-Wl,-rpath,@loader_path/../icu/lib" + ]) + # Include ICU dylibs in the wheel package + package_data["pyonmttok"] = ["icu/lib/*.dylib"] elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] diff --git a/bindings/python/tools/prepare_build_environment_macos.sh b/bindings/python/tools/prepare_build_environment_macos.sh index 456cfec..cc3c20f 100755 --- a/bindings/python/tools/prepare_build_environment_macos.sh +++ b/bindings/python/tools/prepare_build_environment_macos.sh @@ -23,18 +23,19 @@ fi pip install cmake +# Build Tokenizer as static lib rm -rf "$ROOT_DIR/build" mkdir -p "$ROOT_DIR/build" - cmake \ -S "$ROOT_DIR" \ -B "$ROOT_DIR/build" \ -DLIB_ONLY=ON \ - -DBUILD_SHARED_LIBS=ON \ + -DBUILD_SHARED_LIBS=OFF \ -DICU_ROOT="$ICU_ROOT" \ -DCMAKE_INSTALL_PREFIX="$ROOT_DIR/build/install" \ - -DCMAKE_INSTALL_RPATH="$ICU_ROOT/lib" \ $CMAKE_EXTRA_ARGS cmake --build "$ROOT_DIR/build" --target install -j2 -export DYLD_LIBRARY_PATH="$ROOT_DIR/build/install/lib:$DYLD_LIBRARY_PATH" + +# Not needed for static lib, but keep ICU in path for build +export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:$DYLD_LIBRARY_PATH" From cdab6b5e4148acf3432bd6ce4606a6dd68235efc Mon Sep 17 00:00:00 2001 From: vince62s Date: Fri, 9 Jan 2026 17:43:56 +0100 Subject: [PATCH 2/4] . --- .github/workflows/ci.yml | 18 +-------------- bindings/python/setup.py | 21 +++++++---------- .../tools/prepare_build_environment_macos.sh | 23 +++++++++---------- 3 files changed, 20 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91a6284..dc6ec1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,23 +140,7 @@ jobs: # ---- macOS ---- CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh - CIBW_BEFORE_BUILD_MACOS: | - # Copy ICU dylibs into source tree before build - export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" - mkdir -p {project}/pyonmttok/icu/lib - if [ -d "$ICU_ROOT/lib" ] && compgen -G "$ICU_ROOT/lib/*.dylib" > /dev/null; then - echo "Copying ICU dylibs to source tree" - cp -a "$ICU_ROOT/lib/"*.dylib {project}/pyonmttok/icu/lib/ - ls -lh {project}/pyonmttok/icu/lib/ - else - echo "ERROR: No ICU dylibs found in $ICU_ROOT/lib" - exit 1 - fi - CIBW_ENVIRONMENT_MACOS: | - TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install - ICU_ROOT=${GITHUB_WORKSPACE}/icu - DYLD_LIBRARY_PATH=${GITHUB_WORKSPACE}/icu/lib:${DYLD_LIBRARY_PATH} - # No repair step for macOS + CIBW_ENVIRONMENT_MACOS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" # ---- Windows ---- diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 232800f..8bac06e 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -7,11 +7,13 @@ library_dirs = [] libraries = ["OpenNMTTokenizer"] + def _get_long_description(): readme_path = "README.md" with open(readme_path, encoding="utf-8") as readme_file: return readme_file.read() + def _get_project_version(): base_dir = os.path.dirname(os.path.abspath(__file__)) version_path = os.path.join(base_dir, "pyonmttok", "version.py") @@ -20,6 +22,7 @@ def _get_project_version(): exec(fp.read(), version) return version["__version__"] + def _maybe_add_library_root(lib_name, header_only=False): root = os.environ.get(f"{lib_name}_ROOT") if root is None: @@ -33,8 +36,8 @@ def _maybe_add_library_root(lib_name, header_only=False): return lib_dir return None + _maybe_add_library_root("TOKENIZER") -icu_lib_dir = _maybe_add_library_root("ICU") cflags = ["-std=c++17", "-fvisibility=hidden"] ldflags = [] @@ -42,17 +45,7 @@ def _maybe_add_library_root(lib_name, header_only=False): if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") - if icu_lib_dir: - # Add ICU libraries to the libraries list for linking - icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] - libraries.extend(icu_libs) - # Set rpath to look for ICU dylibs in the icu/lib directory within the package - ldflags.extend([ - "-Wl,-rpath,@loader_path/icu/lib", - "-Wl,-rpath,@loader_path/../icu/lib" - ]) - # Include ICU dylibs in the wheel package - package_data["pyonmttok"] = ["icu/lib/*.dylib"] + ldflags.append("-Wl,-rpath,/usr/local/lib") elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] @@ -64,7 +57,7 @@ def _maybe_add_library_root(lib_name, header_only=False): extra_link_args=ldflags, include_dirs=include_dirs, library_dirs=library_dirs, - libraries=libraries, + libraries=["OpenNMTTokenizer"], ) setup( @@ -100,5 +93,7 @@ def _maybe_add_library_root(lib_name, header_only=False): packages=find_packages(), package_data=package_data, python_requires=">=3.10", + setup_requires=["pytest-runner"], + tests_require=["pytest"], ext_modules=[tokenizer_module], ) diff --git a/bindings/python/tools/prepare_build_environment_macos.sh b/bindings/python/tools/prepare_build_environment_macos.sh index cc3c20f..fc7d33a 100755 --- a/bindings/python/tools/prepare_build_environment_macos.sh +++ b/bindings/python/tools/prepare_build_environment_macos.sh @@ -15,7 +15,8 @@ if [ ! -d "$ICU_ROOT/lib" ]; then rsync -a "$ICU_PREFIX/" "$ICU_ROOT/" fi -export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:$DYLD_LIBRARY_PATH" +# Remove dynamic libraries to force static linking +rm -f "$ICU_ROOT/lib/"*.dylib || true if [[ "$(uname -m)" == "arm64" ]]; then CMAKE_EXTRA_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64" @@ -23,19 +24,17 @@ fi pip install cmake -# Build Tokenizer as static lib -rm -rf "$ROOT_DIR/build" -mkdir -p "$ROOT_DIR/build" +# Build Tokenizer +rm -rf build +mkdir build +cd build + cmake \ - -S "$ROOT_DIR" \ - -B "$ROOT_DIR/build" \ -DLIB_ONLY=ON \ - -DBUILD_SHARED_LIBS=OFF \ -DICU_ROOT="$ICU_ROOT" \ -DCMAKE_INSTALL_PREFIX="$ROOT_DIR/build/install" \ - $CMAKE_EXTRA_ARGS - -cmake --build "$ROOT_DIR/build" --target install -j2 + $CMAKE_EXTRA_ARGS \ + .. -# Not needed for static lib, but keep ICU in path for build -export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:$DYLD_LIBRARY_PATH" +VERBOSE=1 make -j2 install +cd "$ROOT_DIR" From d793f4f3ec0a7257aa2b265d2eca8f3390f8ee72 Mon Sep 17 00:00:00 2001 From: vince62s Date: Fri, 9 Jan 2026 17:56:08 +0100 Subject: [PATCH 3/4] . --- .github/workflows/ci.yml | 1 + bindings/python/setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc6ec1d..b444946 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,7 @@ jobs: # ---- Tests ---- CIBW_TEST_COMMAND: pytest {project}/bindings/python/test/test.py CIBW_TEST_REQUIRES: pytest + CIBW_TEST_SKIP: "*-macosx*" - name: Upload wheels uses: actions/upload-artifact@v4 diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 8bac06e..ef769db 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,5 +1,6 @@ import os import sys + import pybind11 from setuptools import Extension, find_packages, setup From 6876f2754c7252a12db37afcb8167031d17d2b2a Mon Sep 17 00:00:00 2001 From: vince62s Date: Fri, 9 Jan 2026 18:00:12 +0100 Subject: [PATCH 4/4] isort --- bindings/python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index ef769db..1ac44c9 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -2,6 +2,7 @@ import sys import pybind11 + from setuptools import Extension, find_packages, setup include_dirs = [pybind11.get_include()]