From d8b9ca4d5a158cf2865d90926b143db1938e179f Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 15:31:20 +0100 Subject: [PATCH 01/19] new CI --- .github/workflows/ci.yml | 174 +++++++++++++++------------------------ 1 file changed, 68 insertions(+), 106 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75abd89..d3a1742 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,145 +2,106 @@ name: CI on: push: - branches: - - master - tags: - - v* + branches: [master] + tags: [v*] pull_request: - branches: - - master + branches: [master] jobs: - build-and-test-cpp: - name: Build and test C++ library on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] +# --------------------------------------------------------- +# 1. C++ library sanity build (fast, no matrix explosion) +# --------------------------------------------------------- + build-cpp: + name: Build C++ library + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - - name: Install ICU libraries on macOS - if: startsWith(matrix.os, 'macos') - run: brew install icu4c - - - name: Build and install - if: startsWith(matrix.os, 'ubuntu') - run: | - cmake -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install . - make install - - - name: Build and install (macOS) - if: startsWith(matrix.os, 'macos') - run: | - ICU_PREFIX=$(brew --prefix icu4c) - cmake \ - -DBUILD_TESTS=ON \ - -DCMAKE_INSTALL_PREFIX=$PWD/install \ - -DICU_ROOT=$ICU_PREFIX \ - . - make install - - - - name: Test + - name: Build C++ library run: | + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON . + make -j2 test/onmt_tokenizer_test test/data - - check-python-style: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install black==22.* flake8==3.9.* isort==5.* - - - name: Check code format with Black - working-directory: bindings/python - run: | - black --check . - - - name: Check imports order with isort - working-directory: bindings/python - run: | - isort --check-only . - - - name: Check code style with Flake8 - working-directory: bindings/python - if: ${{ always() }} - run: | - flake8 . - - - build-and-test-python-wheels: - name: Build and test wheels on ${{ matrix.os }} for ${{ matrix.arch }} +# --------------------------------------------------------- +# 2. Build Python wheels +# --------------------------------------------------------- + build-wheels: + name: Build wheels (${{ matrix.os }} / ${{ matrix.arch }}) runs-on: ${{ matrix.os }} + strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-2019] - arch: [auto64] include: - - os: ubuntu-latest - arch: aarch64 - - os: macos-latest - arch: arm64 + - os: ubuntu-latest + arch: x86_64 + - os: ubuntu-latest + arch: aarch64 + - os: macos-latest + arch: arm64 + - os: windows-latest + arch: amd64 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - - uses: docker/setup-qemu-action@v2 - if: ${{ matrix.arch == 'aarch64' }} - name: Set up QEMU + - uses: docker/setup-qemu-action@v3 + if: matrix.arch == 'aarch64' - name: Build wheels - uses: pypa/cibuildwheel@v2.11.2 + uses: pypa/cibuildwheel@v2.21.0 with: package-dir: bindings/python output-dir: wheelhouse env: - CIBW_ENVIRONMENT_LINUX: TOKENIZER_ROOT=/project/build/install ICU_ROOT=/project/icu - CIBW_ENVIRONMENT_MACOS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install - CIBW_ENVIRONMENT_WINDOWS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install - CIBW_BEFORE_ALL_LINUX: bindings/python/tools/prepare_build_environment_linux.sh - CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh - CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh - CIBW_BEFORE_BUILD: pip install pybind11==2.10.1 - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 CIBW_BUILD: "cp310-* cp311-* cp312-*" + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_SKIP: "pp* *-musllinux_*" + + CIBW_BEFORE_BUILD: pip install pybind11==2.10.1 CIBW_TEST_COMMAND: pytest {project}/bindings/python/test/test.py CIBW_TEST_REQUIRES: pytest - CIBW_ARCHS: ${{ matrix.arch }} - CIBW_SKIP: pp* *-musllinux_* - CIBW_TEST_SKIP: "*-macosx_arm64" + + # Linux + CIBW_ENVIRONMENT_LINUX: > + TOKENIZER_ROOT=/project/build/install + ICU_ROOT=/project/icu + CIBW_BEFORE_ALL_LINUX: bindings/python/tools/prepare_build_environment_linux.sh + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 + + # macOS (arm64 only) + CIBW_ENVIRONMENT_MACOS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install + CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" - - name: Upload Python wheels + # Windows + CIBW_ENVIRONMENT_WINDOWS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install + CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh + + - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: python-wheels-${{ matrix.os }}-${{ matrix.arch }} - path: wheelhouse - - - publish-python-wheels-on-pypi: - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - needs: [build-and-test-cpp, build-and-test-python-wheels] + name: wheels-${{ matrix.os }}-${{ matrix.arch }} + path: wheelhouse/*.whl + +# --------------------------------------------------------- +# 3. Publish to PyPI (tags only) +# --------------------------------------------------------- + publish: + if: startsWith(github.ref, 'refs/tags/v') + needs: [build-cpp, build-wheels] runs-on: ubuntu-latest + permissions: + id-token: write # for Trusted Publishing + steps: - name: Download all wheels uses: actions/download-artifact@v4 @@ -149,10 +110,11 @@ jobs: - name: Collect wheels run: | - mkdir -p dist + mkdir dist find artifacts -name "*.whl" -exec cp {} dist/ \; - - - name: Publish Python wheels to PyPI + ls -lh dist + + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist From ebe03efcd8d3e0cd848b112fa607dd32f88819aa Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 16:27:24 +0100 Subject: [PATCH 02/19] new test --- .github/workflows/ci.yml | 152 +++++++++++++++++++++++---------- bindings/python/pyproject.toml | 8 ++ bindings/python/setup.py | 2 - 3 files changed, 115 insertions(+), 47 deletions(-) create mode 100644 bindings/python/pyproject.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3a1742..de28abd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,102 +8,164 @@ on: branches: [master] jobs: -# --------------------------------------------------------- -# 1. C++ library sanity build (fast, no matrix explosion) -# --------------------------------------------------------- - build-cpp: - name: Build C++ library - runs-on: ubuntu-latest + # ------------------------------------------------------------ + # C++ library build & test (fast sanity check) + # ------------------------------------------------------------ + build-and-test-cpp: + name: Build and test C++ library on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Build C++ library + - name: Install ICU (macOS) + if: runner.os == 'macOS' + run: brew install icu4c + + - name: Build & install (Linux) + if: runner.os == 'Linux' run: | - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON . - make -j2 - test/onmt_tokenizer_test test/data - -# --------------------------------------------------------- -# 2. Build Python wheels -# --------------------------------------------------------- - build-wheels: - name: Build wheels (${{ matrix.os }} / ${{ matrix.arch }}) - runs-on: ${{ matrix.os }} + cmake -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=$PWD/install . + make -j2 install + + - name: Build & install (macOS) + if: runner.os == 'macOS' + run: | + ICU_PREFIX=$(brew --prefix icu4c) + cmake \ + -DBUILD_TESTS=ON \ + -DCMAKE_INSTALL_PREFIX=$PWD/install \ + -DICU_ROOT=$ICU_PREFIX \ + . + make -j2 install + + - name: Run C++ tests + run: test/onmt_tokenizer_test test/data + + + # ------------------------------------------------------------ + # Python style checks + # ------------------------------------------------------------ + check-python-style: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install linters + run: | + pip install black==22.* flake8==3.9.* isort==5.* + + - name: Black + working-directory: bindings/python + run: black --check . + - name: isort + working-directory: bindings/python + run: isort --check-only . + + - name: Flake8 + working-directory: bindings/python + run: flake8 . + + + # ------------------------------------------------------------ + # Build Python wheels + # ------------------------------------------------------------ + build-python-wheels: + name: Wheels – ${{ matrix.os }} / ${{ matrix.arch }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: + # Linux - os: ubuntu-latest arch: x86_64 - os: ubuntu-latest arch: aarch64 + + # macOS + - os: macos-latest + arch: x86_64 - os: macos-latest arch: arm64 - - os: windows-latest - arch: amd64 + + # Windows + - os: windows-2019 + arch: AMD64 steps: - uses: actions/checkout@v4 with: submodules: recursive - - uses: docker/setup-qemu-action@v3 + - name: Enable QEMU (Linux ARM) if: matrix.arch == 'aarch64' + uses: docker/setup-qemu-action@v3 - name: Build wheels - uses: pypa/cibuildwheel@v2.21.0 + uses: pypa/cibuildwheel@v2.11.2 with: package-dir: bindings/python output-dir: wheelhouse env: + # ---- Build selection ---- CIBW_BUILD: "cp310-* cp311-* cp312-*" - CIBW_ARCHS: ${{ matrix.arch }} CIBW_SKIP: "pp* *-musllinux_*" + CIBW_ARCHS: ${{ matrix.arch }} + # ---- Dependencies ---- CIBW_BEFORE_BUILD: pip install pybind11==2.10.1 - CIBW_TEST_COMMAND: pytest {project}/bindings/python/test/test.py - CIBW_TEST_REQUIRES: pytest - # Linux - CIBW_ENVIRONMENT_LINUX: > + # ---- Linux ---- + CIBW_BEFORE_ALL_LINUX: bindings/python/tools/prepare_build_environment_linux.sh + CIBW_ENVIRONMENT_LINUX: | TOKENIZER_ROOT=/project/build/install ICU_ROOT=/project/icu - CIBW_BEFORE_ALL_LINUX: bindings/python/tools/prepare_build_environment_linux.sh CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 - # macOS (arm64 only) - CIBW_ENVIRONMENT_MACOS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install + # ---- macOS ---- CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh + CIBW_ENVIRONMENT_MACOS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" - # Windows - CIBW_ENVIRONMENT_WINDOWS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install + # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh + CIBW_ENVIRONMENT_WINDOWS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install + + # ---- Tests ---- + CIBW_TEST_COMMAND: pytest {project}/bindings/python/test/test.py + CIBW_TEST_REQUIRES: pytest - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.arch }} - path: wheelhouse/*.whl - -# --------------------------------------------------------- -# 3. Publish to PyPI (tags only) -# --------------------------------------------------------- - publish: - if: startsWith(github.ref, 'refs/tags/v') - needs: [build-cpp, build-wheels] - runs-on: ubuntu-latest + name: python-wheels-${{ matrix.os }}-${{ matrix.arch }} + path: wheelhouse - permissions: - id-token: write # for Trusted Publishing + + # ------------------------------------------------------------ + # Publish to PyPI (API token, no Trusted Publishing) + # ------------------------------------------------------------ + publish-python-wheels: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + needs: [build-and-test-cpp, build-python-wheels] + runs-on: ubuntu-latest steps: - - name: Download all wheels + - name: Download wheels uses: actions/download-artifact@v4 with: path: artifacts @@ -112,7 +174,6 @@ jobs: run: | mkdir dist find artifacts -name "*.whl" -exec cp {} dist/ \; - ls -lh dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 @@ -120,3 +181,4 @@ jobs: packages-dir: dist user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml new file mode 100644 index 0000000..fadeeb8 --- /dev/null +++ b/bindings/python/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "setuptools>=61", + "wheel", + "pybind11==2.10.1" +] +build-backend = "setuptools.build_meta" + diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 178eda8..58c6679 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -92,7 +92,5 @@ def _maybe_add_library_root(lib_name, header_only=False): packages=find_packages(), package_data=package_data, python_requires=">=3.10", - setup_requires=["pytest-runner"], - tests_require=["pytest"], ext_modules=[tokenizer_module], ) From bc8424a88dc530e7a15362ab02f99ce475049174 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 16:40:42 +0100 Subject: [PATCH 03/19] fix macos --- bindings/python/setup.py | 2 +- bindings/python/tools/prepare_build_environment_macos.sh | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 58c6679..6ec3b3d 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -44,7 +44,7 @@ def _maybe_add_library_root(lib_name, header_only=False): package_data = {} if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") - ldflags.append("-Wl,-rpath,/usr/local/lib") + ldflags.append("-Wl,-rpath,@loader_path") elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] diff --git a/bindings/python/tools/prepare_build_environment_macos.sh b/bindings/python/tools/prepare_build_environment_macos.sh index 773783e..4dba2dd 100755 --- a/bindings/python/tools/prepare_build_environment_macos.sh +++ b/bindings/python/tools/prepare_build_environment_macos.sh @@ -16,9 +16,6 @@ ICU_PREFIX="$(brew --prefix icu4c)" # Copy ICU into local prefix rsync -a "$ICU_PREFIX/" "$ICU_ROOT/" -# Remove dynamic libraries to force static linking -rm -f "$ICU_ROOT/lib/"*.dylib || true - # Explicit Apple Silicon handling if [[ "$(uname -m)" == "arm64" ]]; then CMAKE_EXTRA_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64" @@ -27,12 +24,13 @@ fi # Install cmake pip install cmake -# Build Tokenizer +# Build Tokenizer as static library rm -rf build mkdir build cd build cmake \ -DLIB_ONLY=ON \ + -DBUILD_SHARED_LIBS=OFF \ # << STATIC LIB -DICU_ROOT="$ICU_ROOT" \ -DCMAKE_INSTALL_PREFIX="$ROOT_DIR/build/install" \ $CMAKE_EXTRA_ARGS \ From b6b1b9cae611a5e9b54ec6cd5a513c28a9f3639d Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 16:45:06 +0100 Subject: [PATCH 04/19] fix --- .../tools/prepare_build_environment_macos.sh | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/bindings/python/tools/prepare_build_environment_macos.sh b/bindings/python/tools/prepare_build_environment_macos.sh index 4dba2dd..f6173df 100755 --- a/bindings/python/tools/prepare_build_environment_macos.sh +++ b/bindings/python/tools/prepare_build_environment_macos.sh @@ -1,5 +1,4 @@ -#! /bin/bash - +#!/bin/bash set -e set -x @@ -9,33 +8,28 @@ CMAKE_EXTRA_ARGS="" mkdir -p "$ICU_ROOT" -# Install ICU via Homebrew brew install icu4c ICU_PREFIX="$(brew --prefix icu4c)" - -# Copy ICU into local prefix rsync -a "$ICU_PREFIX/" "$ICU_ROOT/" -# Explicit Apple Silicon handling if [[ "$(uname -m)" == "arm64" ]]; then CMAKE_EXTRA_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64" fi -# Install cmake pip install cmake -# Build Tokenizer as static library -rm -rf build -mkdir build -cd build +# Build Tokenizer as static lib +rm -rf "$ROOT_DIR/build" +mkdir -p "$ROOT_DIR/build" + cmake \ + -S "$ROOT_DIR" \ + -B "$ROOT_DIR/build" \ -DLIB_ONLY=ON \ - -DBUILD_SHARED_LIBS=OFF \ # << STATIC LIB + -DBUILD_SHARED_LIBS=OFF \ -DICU_ROOT="$ICU_ROOT" \ -DCMAKE_INSTALL_PREFIX="$ROOT_DIR/build/install" \ - $CMAKE_EXTRA_ARGS \ - .. + $CMAKE_EXTRA_ARGS -VERBOSE=1 make -j2 install -cd "$ROOT_DIR" +cmake --build "$ROOT_DIR/build" --target install -j2 From 28c3f40f93a4bd6d23fe2e5cf680d135359e7cf2 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 16:57:49 +0100 Subject: [PATCH 05/19] another try --- .github/workflows/ci.yml | 14 ++++++++++++-- bindings/python/setup.py | 6 +++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de28abd..c9f5e12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,8 +138,18 @@ jobs: # ---- macOS ---- CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh - CIBW_ENVIRONMENT_MACOS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install - CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + CIBW_ENVIRONMENT_MACOS: | + TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install + ICU_ROOT=${GITHUB_WORKSPACE}/icu # ensure ICU path is exported + CIBW_REPAIR_WHEEL_COMMAND_MACOS: | + # Copy ICU dylibs into wheel package so rpath works + for whl in dist/*.whl; do + tmpdir=$(mktemp -d) + unzip -q "$whl" -d "$tmpdir" + mkdir -p "$tmpdir/pyonmttok/icu/lib" + cp -a $ICU_ROOT/lib/*.dylib "$tmpdir/pyonmttok/icu/lib/" + (cd "$tmpdir" && zip -qr "$GITHUB_WORKSPACE/wheelhouse/$(basename "$whl")" .) + done # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 6ec3b3d..54cc3e1 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -44,7 +44,7 @@ def _maybe_add_library_root(lib_name, header_only=False): package_data = {} if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") - ldflags.append("-Wl,-rpath,@loader_path") + ldflags.append("-Wl,-rpath,@loader_path/../icu/lib") elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] @@ -59,6 +59,10 @@ def _maybe_add_library_root(lib_name, header_only=False): libraries=["OpenNMTTokenizer"], ) +# Include ICU dylibs in the wheel for macOS +if sys.platform == "darwin": + package_data["pyonmttok"] = ["icu/lib/*.dylib"] + setup( name="pyonmttok", version=_get_project_version(), From b7154c24eccfed5a3b7a65d343d6ee7559e77cad Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:07:50 +0100 Subject: [PATCH 06/19] repair not working --- .github/workflows/ci.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9f5e12..27954b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,14 +142,9 @@ jobs: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install ICU_ROOT=${GITHUB_WORKSPACE}/icu # ensure ICU path is exported CIBW_REPAIR_WHEEL_COMMAND_MACOS: | - # Copy ICU dylibs into wheel package so rpath works - for whl in dist/*.whl; do - tmpdir=$(mktemp -d) - unzip -q "$whl" -d "$tmpdir" - mkdir -p "$tmpdir/pyonmttok/icu/lib" - cp -a $ICU_ROOT/lib/*.dylib "$tmpdir/pyonmttok/icu/lib/" - (cd "$tmpdir" && zip -qr "$GITHUB_WORKSPACE/wheelhouse/$(basename "$whl")" .) - done + # Copy ICU dylibs into wheel folder + mkdir -p pyonmttok/icu/lib + cp -a "$ICU_ROOT/lib/"*.dylib pyonmttok/icu/lib/ # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh From c5879b24a5cac75c170a6886c94829bf984211d3 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:16:13 +0100 Subject: [PATCH 07/19] macos sucks --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27954b3..d66199c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,9 +142,13 @@ jobs: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install ICU_ROOT=${GITHUB_WORKSPACE}/icu # ensure ICU path is exported CIBW_REPAIR_WHEEL_COMMAND_MACOS: | - # Copy ICU dylibs into wheel folder - mkdir -p pyonmttok/icu/lib - cp -a "$ICU_ROOT/lib/"*.dylib pyonmttok/icu/lib/ + ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" + if compgen -G "$ICU_ROOT/lib/*.dylib" > /dev/null; then + mkdir -p pyonmttok/icu/lib + cp -a "$ICU_ROOT/lib/"*.dylib pyonmttok/icu/lib/ + else + echo "Warning: No ICU dylibs found in $ICU_ROOT/lib. Skipping copy." + fi # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh From 86722d12d4686089a0d1734253ca402f87a2e585 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:24:23 +0100 Subject: [PATCH 08/19] claude better? --- .github/workflows/ci.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d66199c..b1d3712 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,15 +140,29 @@ jobs: CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh CIBW_ENVIRONMENT_MACOS: | TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install - ICU_ROOT=${GITHUB_WORKSPACE}/icu # ensure ICU path is exported + ICU_ROOT=${GITHUB_WORKSPACE}/icu CIBW_REPAIR_WHEEL_COMMAND_MACOS: | - ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" - if compgen -G "$ICU_ROOT/lib/*.dylib" > /dev/null; then - mkdir -p pyonmttok/icu/lib - cp -a "$ICU_ROOT/lib/"*.dylib pyonmttok/icu/lib/ + export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" + + # Unpack wheel + wheel unpack {wheel} + WHEEL_DIR=$(find . -maxdepth 1 -type d -name "pyonmttok-*" | head -n 1) + + # Copy ICU dylibs into the unpacked wheel + if [ -d "$ICU_ROOT/lib" ] && compgen -G "$ICU_ROOT/lib/*.dylib" > /dev/null; then + echo "Copying ICU dylibs from $ICU_ROOT/lib" + mkdir -p "$WHEEL_DIR/pyonmttok/icu/lib" + cp -a "$ICU_ROOT/lib/"*.dylib "$WHEEL_DIR/pyonmttok/icu/lib/" else - echo "Warning: No ICU dylibs found in $ICU_ROOT/lib. Skipping copy." + echo "Warning: No ICU dylibs found in $ICU_ROOT/lib" fi + + # Repack wheel + wheel pack "$WHEEL_DIR" -d {dest_dir} + rm -rf "$WHEEL_DIR" + + # Run delocate to fix library paths + delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {dest_dir}/*.whl # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh @@ -190,4 +204,3 @@ jobs: packages-dir: dist user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - From 93936cb61e60c07f48c1692c839bf6d0143ae3a0 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:30:44 +0100 Subject: [PATCH 09/19] ... --- .github/workflows/ci.yml | 29 +++++++++++++++++------------ bindings/python/setup.py | 13 +++++++------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1d3712..b3ecd68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,25 +144,30 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_MACOS: | export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" - # Unpack wheel - wheel unpack {wheel} - WHEEL_DIR=$(find . -maxdepth 1 -type d -name "pyonmttok-*" | head -n 1) + # First run delocate to bundle dependencies + delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} + + # Then add ICU dylibs to the repaired wheel + REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) - # Copy ICU dylibs into the unpacked wheel if [ -d "$ICU_ROOT/lib" ] && compgen -G "$ICU_ROOT/lib/*.dylib" > /dev/null; then - echo "Copying ICU dylibs from $ICU_ROOT/lib" + echo "Adding ICU dylibs to repaired wheel" + + # Unpack the repaired wheel + wheel unpack "$REPAIRED_WHEEL" + WHEEL_DIR=$(find . -maxdepth 1 -type d -name "pyonmttok-*" | head -n 1) + + # Copy ICU dylibs mkdir -p "$WHEEL_DIR/pyonmttok/icu/lib" cp -a "$ICU_ROOT/lib/"*.dylib "$WHEEL_DIR/pyonmttok/icu/lib/" + + # Repack wheel (overwrite the repaired one) + rm "$REPAIRED_WHEEL" + wheel pack "$WHEEL_DIR" -d {dest_dir} + rm -rf "$WHEEL_DIR" else echo "Warning: No ICU dylibs found in $ICU_ROOT/lib" fi - - # Repack wheel - wheel pack "$WHEEL_DIR" -d {dest_dir} - rm -rf "$WHEEL_DIR" - - # Run delocate to fix library paths - delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {dest_dir}/*.whl # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 54cc3e1..7ff4c3a 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -2,7 +2,6 @@ import sys import pybind11 - from setuptools import Extension, find_packages, setup include_dirs = [pybind11.get_include()] @@ -42,9 +41,15 @@ def _maybe_add_library_root(lib_name, header_only=False): cflags = ["-std=c++17", "-fvisibility=hidden"] ldflags = [] package_data = {} + if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") - ldflags.append("-Wl,-rpath,@loader_path/../icu/lib") + # Set rpath to look for ICU libraries in the icu/lib directory + ldflags.extend( + ["-Wl,-rpath,@loader_path/icu/lib", "-Wl,-rpath,@loader_path/../icu/lib"] + ) + # Include ICU dylibs in the wheel + package_data["pyonmttok"] = ["icu/lib/*.dylib"] elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] @@ -59,10 +64,6 @@ def _maybe_add_library_root(lib_name, header_only=False): libraries=["OpenNMTTokenizer"], ) -# Include ICU dylibs in the wheel for macOS -if sys.platform == "darwin": - package_data["pyonmttok"] = ["icu/lib/*.dylib"] - setup( name="pyonmttok", version=_get_project_version(), From 1e55208688be878267213ebfd56bf7a14a9919ba Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:37:06 +0100 Subject: [PATCH 10/19] ... --- .github/workflows/ci.yml | 26 +++----------------------- bindings/python/setup.py | 11 +---------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3ecd68..d69acac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,33 +141,13 @@ jobs: CIBW_ENVIRONMENT_MACOS: | TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install ICU_ROOT=${GITHUB_WORKSPACE}/icu + DYLD_LIBRARY_PATH=${GITHUB_WORKSPACE}/icu/lib:${DYLD_LIBRARY_PATH} CIBW_REPAIR_WHEEL_COMMAND_MACOS: | export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" + export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:${DYLD_LIBRARY_PATH}" - # First run delocate to bundle dependencies + # Run delocate with the ICU library path so it can find and bundle ICU delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} - - # Then add ICU dylibs to the repaired wheel - REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) - - if [ -d "$ICU_ROOT/lib" ] && compgen -G "$ICU_ROOT/lib/*.dylib" > /dev/null; then - echo "Adding ICU dylibs to repaired wheel" - - # Unpack the repaired wheel - wheel unpack "$REPAIRED_WHEEL" - WHEEL_DIR=$(find . -maxdepth 1 -type d -name "pyonmttok-*" | head -n 1) - - # Copy ICU dylibs - mkdir -p "$WHEEL_DIR/pyonmttok/icu/lib" - cp -a "$ICU_ROOT/lib/"*.dylib "$WHEEL_DIR/pyonmttok/icu/lib/" - - # Repack wheel (overwrite the repaired one) - rm "$REPAIRED_WHEEL" - wheel pack "$WHEEL_DIR" -d {dest_dir} - rm -rf "$WHEEL_DIR" - else - echo "Warning: No ICU dylibs found in $ICU_ROOT/lib" - fi # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 7ff4c3a..d1a40ac 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -7,13 +7,11 @@ include_dirs = [pybind11.get_include()] library_dirs = [] - def _get_long_description(): readme_path = "README.md" with open(readme_path, encoding="utf-8") as readme_file: return readme_file.read() - def _get_project_version(): base_dir = os.path.dirname(os.path.abspath(__file__)) version_path = os.path.join(base_dir, "pyonmttok", "version.py") @@ -22,7 +20,6 @@ def _get_project_version(): exec(fp.read(), version) return version["__version__"] - def _maybe_add_library_root(lib_name, header_only=False): root = os.environ.get("%s_ROOT" % lib_name) if root is None: @@ -35,8 +32,8 @@ def _maybe_add_library_root(lib_name, header_only=False): library_dirs.append(lib_dir) break - _maybe_add_library_root("TOKENIZER") +_maybe_add_library_root("ICU") cflags = ["-std=c++17", "-fvisibility=hidden"] ldflags = [] @@ -44,12 +41,6 @@ def _maybe_add_library_root(lib_name, header_only=False): if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") - # Set rpath to look for ICU libraries in the icu/lib directory - ldflags.extend( - ["-Wl,-rpath,@loader_path/icu/lib", "-Wl,-rpath,@loader_path/../icu/lib"] - ) - # Include ICU dylibs in the wheel - package_data["pyonmttok"] = ["icu/lib/*.dylib"] elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] From 67f30851da9fcc4322237250cbd4aeef1ce7dc3a Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:42:02 +0100 Subject: [PATCH 11/19] ... --- .github/workflows/ci.yml | 21 ++++++++++++++++++++- bindings/python/setup.py | 11 ++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d69acac..f001e6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,8 +146,27 @@ jobs: export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:${DYLD_LIBRARY_PATH}" - # Run delocate with the ICU library path so it can find and bundle ICU + echo "=== Checking wheel dependencies before repair ===" + unzip -l {wheel} + otool -L {wheel}/*.so 2>/dev/null || true + + echo "=== ICU libraries available at $ICU_ROOT/lib ===" + ls -la "$ICU_ROOT/lib/" || true + + echo "=== Running delocate-wheel ===" delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} + + echo "=== Checking repaired wheel ===" + REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) + unzip -l "$REPAIRED_WHEEL" + + echo "=== Checking bundled libraries ===" + unzip -q "$REPAIRED_WHEEL" -d /tmp/wheel_check + find /tmp/wheel_check -name "*.dylib" -o -name "*.so" | while read lib; do + echo "Library: $lib" + otool -L "$lib" || true + done + rm -rf /tmp/wheel_check # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh diff --git a/bindings/python/setup.py b/bindings/python/setup.py index d1a40ac..dc63d71 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -2,16 +2,20 @@ import sys import pybind11 + from setuptools import Extension, find_packages, setup include_dirs = [pybind11.get_include()] library_dirs = [] +libraries = ["OpenNMTTokenizer"] + def _get_long_description(): readme_path = "README.md" with open(readme_path, encoding="utf-8") as readme_file: return readme_file.read() + def _get_project_version(): base_dir = os.path.dirname(os.path.abspath(__file__)) version_path = os.path.join(base_dir, "pyonmttok", "version.py") @@ -20,6 +24,7 @@ def _get_project_version(): exec(fp.read(), version) return version["__version__"] + def _maybe_add_library_root(lib_name, header_only=False): root = os.environ.get("%s_ROOT" % lib_name) if root is None: @@ -32,6 +37,7 @@ def _maybe_add_library_root(lib_name, header_only=False): library_dirs.append(lib_dir) break + _maybe_add_library_root("TOKENIZER") _maybe_add_library_root("ICU") @@ -41,6 +47,9 @@ def _maybe_add_library_root(lib_name, header_only=False): if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") + # Explicitly link ICU libraries + icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] + libraries.extend(icu_libs) elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] @@ -52,7 +61,7 @@ def _maybe_add_library_root(lib_name, header_only=False): extra_link_args=ldflags, include_dirs=include_dirs, library_dirs=library_dirs, - libraries=["OpenNMTTokenizer"], + libraries=libraries, ) setup( From c80178d2f438c8839b287860137eac2cb76a3501 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:46:22 +0100 Subject: [PATCH 12/19] . --- bindings/python/setup.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index dc63d71..d8b2b34 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -28,18 +28,19 @@ def _get_project_version(): def _maybe_add_library_root(lib_name, header_only=False): root = os.environ.get("%s_ROOT" % lib_name) if root is None: - return + return None include_dirs.append(os.path.join(root, "include")) if not header_only: for lib_subdir in ("lib64", "lib"): lib_dir = os.path.join(root, lib_subdir) if os.path.isdir(lib_dir): library_dirs.append(lib_dir) - break + return lib_dir + return None _maybe_add_library_root("TOKENIZER") -_maybe_add_library_root("ICU") +icu_lib_dir = _maybe_add_library_root("ICU") cflags = ["-std=c++17", "-fvisibility=hidden"] ldflags = [] @@ -47,9 +48,12 @@ def _maybe_add_library_root(lib_name, header_only=False): if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") - # Explicitly link ICU libraries - icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] - libraries.extend(icu_libs) + # Explicitly link ICU libraries if ICU_ROOT is set + if icu_lib_dir: + icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] + libraries.extend(icu_libs) + # Add rpath for the linker to find ICU during build + ldflags.append(f"-Wl,-rpath,{icu_lib_dir}") elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] From a36bdfeb14bbb32c4f6bab7ff4899d9d753e7abf Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 17:50:23 +0100 Subject: [PATCH 13/19] . --- .github/workflows/ci.yml | 55 +++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f001e6f..d2dce5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,27 +146,52 @@ jobs: export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:${DYLD_LIBRARY_PATH}" - echo "=== Checking wheel dependencies before repair ===" - unzip -l {wheel} - otool -L {wheel}/*.so 2>/dev/null || true + echo "=== Extracting wheel to check dependencies ===" + mkdir -p /tmp/wheel_inspect + cd /tmp/wheel_inspect + unzip -q {wheel} - echo "=== ICU libraries available at $ICU_ROOT/lib ===" - ls -la "$ICU_ROOT/lib/" || true + echo "=== Finding .so file ===" + SO_FILE=$(find . -name "_ext*.so" | head -n 1) + echo "Found: $SO_FILE" - echo "=== Running delocate-wheel ===" - delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} + echo "=== Dependencies of $SO_FILE ===" + otool -L "$SO_FILE" + + echo "=== ICU libraries available ===" + ls -lh "$ICU_ROOT/lib/"libicu* 2>/dev/null || echo "No ICU libs found" + + cd - + rm -rf /tmp/wheel_inspect + + echo "=== Running delocate-listdeps to see what it detects ===" + delocate-listdeps {wheel} + + echo "=== Running delocate-wheel with library search path ===" + DYLD_LIBRARY_PATH="$ICU_ROOT/lib:${DYLD_LIBRARY_PATH}" \ + delocate-wheel --require-archs {delocate_archs} \ + --dylibs-only -w {dest_dir} -v {wheel} echo "=== Checking repaired wheel ===" REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) - unzip -l "$REPAIRED_WHEEL" + echo "Repaired wheel: $REPAIRED_WHEEL" + + mkdir -p /tmp/repaired_check + cd /tmp/repaired_check + unzip -q "$REPAIRED_WHEEL" + + echo "=== Contents of repaired wheel ===" + find . -type f -name "*.dylib" -o -name "*.so" + + echo "=== Checking dependencies in repaired wheel ===" + SO_FILE=$(find . -name "_ext*.so" | head -n 1) + if [ -n "$SO_FILE" ]; then + echo "Dependencies of repaired $SO_FILE:" + otool -L "$SO_FILE" + fi - echo "=== Checking bundled libraries ===" - unzip -q "$REPAIRED_WHEEL" -d /tmp/wheel_check - find /tmp/wheel_check -name "*.dylib" -o -name "*.so" | while read lib; do - echo "Library: $lib" - otool -L "$lib" || true - done - rm -rf /tmp/wheel_check + cd - + rm -rf /tmp/repaired_check # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh From 1e01bd7aa4f516b376b0635dd51c655f9026dce7 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 18:12:27 +0100 Subject: [PATCH 14/19] . --- bindings/python/tools/prepare_build_environment_macos.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bindings/python/tools/prepare_build_environment_macos.sh b/bindings/python/tools/prepare_build_environment_macos.sh index f6173df..77dbee9 100755 --- a/bindings/python/tools/prepare_build_environment_macos.sh +++ b/bindings/python/tools/prepare_build_environment_macos.sh @@ -11,6 +11,7 @@ mkdir -p "$ICU_ROOT" brew install icu4c ICU_PREFIX="$(brew --prefix icu4c)" rsync -a "$ICU_PREFIX/" "$ICU_ROOT/" +export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:$DYLD_LIBRARY_PATH" if [[ "$(uname -m)" == "arm64" ]]; then CMAKE_EXTRA_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64" @@ -29,6 +30,9 @@ cmake \ -DBUILD_SHARED_LIBS=OFF \ -DICU_ROOT="$ICU_ROOT" \ -DCMAKE_INSTALL_PREFIX="$ROOT_DIR/build/install" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_MACOSX_RPATH=ON \ + -DCMAKE_INSTALL_RPATH="$ICU_ROOT/lib" \ $CMAKE_EXTRA_ARGS cmake --build "$ROOT_DIR/build" --target install -j2 From ddb604154a8866d1c97a2b90617e590a3bfaf967 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 18:35:45 +0100 Subject: [PATCH 15/19] . --- .github/workflows/ci.yml | 92 ++++++++++++++++++++-------------------- bindings/python/setup.py | 4 +- 2 files changed, 46 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2dce5b..9b9f63a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,55 +143,53 @@ jobs: ICU_ROOT=${GITHUB_WORKSPACE}/icu DYLD_LIBRARY_PATH=${GITHUB_WORKSPACE}/icu/lib:${DYLD_LIBRARY_PATH} CIBW_REPAIR_WHEEL_COMMAND_MACOS: | - export ICU_ROOT="${ICU_ROOT:-${GITHUB_WORKSPACE}/icu}" - export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:${DYLD_LIBRARY_PATH}" - - echo "=== Extracting wheel to check dependencies ===" - mkdir -p /tmp/wheel_inspect - cd /tmp/wheel_inspect - unzip -q {wheel} - - echo "=== Finding .so file ===" - SO_FILE=$(find . -name "_ext*.so" | head -n 1) - echo "Found: $SO_FILE" - - echo "=== Dependencies of $SO_FILE ===" + set -e + + # Path to ICU libraries + ICU_LIB_DIR="${ICU_ROOT}/lib" + + echo "=== Wheel repair start ===" + echo "Wheel: {wheel}" + echo "Destination dir: {dest_dir}" + echo "ICU library path: $ICU_LIB_DIR" + + # Extract wheel for inspection + TEMP_DIR=$(mktemp -d) + unzip -q {wheel} -d "$TEMP_DIR" + cd "$TEMP_DIR" + + # Find the .so extension module + SO_FILE=$(find . -name "_ext*.so" | head -n1) + if [ -z "$SO_FILE" ]; then + echo "Error: no _ext*.so found in wheel!" + exit 1 + fi + echo "Found extension: $SO_FILE" + + echo "=== Inspect original dependencies ===" otool -L "$SO_FILE" - - echo "=== ICU libraries available ===" - ls -lh "$ICU_ROOT/lib/"libicu* 2>/dev/null || echo "No ICU libs found" - - cd - - rm -rf /tmp/wheel_inspect - - echo "=== Running delocate-listdeps to see what it detects ===" - delocate-listdeps {wheel} - - echo "=== Running delocate-wheel with library search path ===" - DYLD_LIBRARY_PATH="$ICU_ROOT/lib:${DYLD_LIBRARY_PATH}" \ - delocate-wheel --require-archs {delocate_archs} \ - --dylibs-only -w {dest_dir} -v {wheel} - - echo "=== Checking repaired wheel ===" + + # Fix rpath: use @loader_path so ICU dylibs are relative to the wheel + echo "=== Adding @loader_path rpath ===" + install_name_tool -add_rpath @loader_path/../icu/lib "$SO_FILE" + + # Copy ICU dylibs into the wheel if not already present + mkdir -p $(dirname "$SO_FILE")/../icu/lib + cp "$ICU_LIB_DIR"/libicu*.dylib $(dirname "$SO_FILE")/../icu/lib/ || true + + echo "=== Running delocate to bundle ICU ===" + delocate-wheel -v -w {dest_dir} {wheel} + + echo "=== Inspect repaired wheel ===" REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) - echo "Repaired wheel: $REPAIRED_WHEEL" - - mkdir -p /tmp/repaired_check - cd /tmp/repaired_check - unzip -q "$REPAIRED_WHEEL" - - echo "=== Contents of repaired wheel ===" - find . -type f -name "*.dylib" -o -name "*.so" - - echo "=== Checking dependencies in repaired wheel ===" - SO_FILE=$(find . -name "_ext*.so" | head -n 1) - if [ -n "$SO_FILE" ]; then - echo "Dependencies of repaired $SO_FILE:" - otool -L "$SO_FILE" - fi - - cd - - rm -rf /tmp/repaired_check + unzip -q "$REPAIRED_WHEEL" -d /tmp/wheel_check + SO_REPAIRED=$(find /tmp/wheel_check -name "_ext*.so" | head -n1) + echo "Dependencies of repaired extension:" + otool -L "$SO_REPAIRED" + rm -rf /tmp/wheel_check + + echo "=== Wheel repair done: $REPAIRED_WHEEL ===" + # ---- Windows ---- CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh diff --git a/bindings/python/setup.py b/bindings/python/setup.py index d8b2b34..6eeaeed 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -48,12 +48,10 @@ def _maybe_add_library_root(lib_name, header_only=False): if sys.platform == "darwin": cflags.append("-mmacosx-version-min=10.14") - # Explicitly link ICU libraries if ICU_ROOT is set if icu_lib_dir: icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] libraries.extend(icu_libs) - # Add rpath for the linker to find ICU during build - ldflags.append(f"-Wl,-rpath,{icu_lib_dir}") + ldflags.append(f"-Wl,-rpath,@loader_path/../icu/lib") elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] From 36d345b8ed3252326fc252206e71e58e87c226e5 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 18:51:22 +0100 Subject: [PATCH 16/19] . --- .github/workflows/ci.yml | 42 ++++--------------- bindings/python/setup.py | 8 +++- .../tools/prepare_build_environment_macos.sh | 11 +++-- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b9f63a..c1bd290 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,50 +145,22 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_MACOS: | set -e - # Path to ICU libraries - ICU_LIB_DIR="${ICU_ROOT}/lib" - - echo "=== Wheel repair start ===" + echo "=== Starting wheel repair ===" echo "Wheel: {wheel}" - echo "Destination dir: {dest_dir}" - echo "ICU library path: $ICU_LIB_DIR" - - # Extract wheel for inspection - TEMP_DIR=$(mktemp -d) - unzip -q {wheel} -d "$TEMP_DIR" - cd "$TEMP_DIR" - - # Find the .so extension module - SO_FILE=$(find . -name "_ext*.so" | head -n1) - if [ -z "$SO_FILE" ]; then - echo "Error: no _ext*.so found in wheel!" - exit 1 - fi - echo "Found extension: $SO_FILE" - - echo "=== Inspect original dependencies ===" - otool -L "$SO_FILE" - - # Fix rpath: use @loader_path so ICU dylibs are relative to the wheel - echo "=== Adding @loader_path rpath ===" - install_name_tool -add_rpath @loader_path/../icu/lib "$SO_FILE" + echo "Destination: {dest_dir}" + echo "ICU path: ${ICU_ROOT}/lib" - # Copy ICU dylibs into the wheel if not already present - mkdir -p $(dirname "$SO_FILE")/../icu/lib - cp "$ICU_LIB_DIR"/libicu*.dylib $(dirname "$SO_FILE")/../icu/lib/ || true - - echo "=== Running delocate to bundle ICU ===" + # Use delocate to bundle ICU dylibs into the wheel delocate-wheel -v -w {dest_dir} {wheel} echo "=== Inspect repaired wheel ===" REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) unzip -q "$REPAIRED_WHEEL" -d /tmp/wheel_check - SO_REPAIRED=$(find /tmp/wheel_check -name "_ext*.so" | head -n1) - echo "Dependencies of repaired extension:" - otool -L "$SO_REPAIRED" + SO_FILE=$(find /tmp/wheel_check -name "_ext*.so" | head -n1) + otool -L "$SO_FILE" rm -rf /tmp/wheel_check + echo "=== Wheel repair complete: $REPAIRED_WHEEL ===" - echo "=== Wheel repair done: $REPAIRED_WHEEL ===" # ---- Windows ---- diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 6eeaeed..0fd1aa0 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -26,7 +26,7 @@ def _get_project_version(): def _maybe_add_library_root(lib_name, header_only=False): - root = os.environ.get("%s_ROOT" % lib_name) + root = os.environ.get(f"{lib_name}_ROOT") if root is None: return None include_dirs.append(os.path.join(root, "include")) @@ -51,7 +51,13 @@ def _maybe_add_library_root(lib_name, header_only=False): if icu_lib_dir: icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] libraries.extend(icu_libs) + + # Ensure _ext uses relative rpath to wheel ICU ldflags.append(f"-Wl,-rpath,@loader_path/../icu/lib") + # Force linking against ICU libraries in ICU_ROOT + ldflags.extend( + [f"-L{icu_lib_dir}", "-licuuc", "-licudata", "-licui18n", "-licuio"] + ) elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] diff --git a/bindings/python/tools/prepare_build_environment_macos.sh b/bindings/python/tools/prepare_build_environment_macos.sh index 77dbee9..e030ce4 100755 --- a/bindings/python/tools/prepare_build_environment_macos.sh +++ b/bindings/python/tools/prepare_build_environment_macos.sh @@ -8,9 +8,13 @@ CMAKE_EXTRA_ARGS="" mkdir -p "$ICU_ROOT" -brew install icu4c -ICU_PREFIX="$(brew --prefix icu4c)" -rsync -a "$ICU_PREFIX/" "$ICU_ROOT/" +# Only copy ICU if not already present +if [ ! -d "$ICU_ROOT/lib" ]; then + brew install icu4c + ICU_PREFIX="$(brew --prefix icu4c)" + rsync -a "$ICU_PREFIX/" "$ICU_ROOT/" +fi + export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:$DYLD_LIBRARY_PATH" if [[ "$(uname -m)" == "arm64" ]]; then @@ -30,7 +34,6 @@ cmake \ -DBUILD_SHARED_LIBS=OFF \ -DICU_ROOT="$ICU_ROOT" \ -DCMAKE_INSTALL_PREFIX="$ROOT_DIR/build/install" \ - -DCMAKE_OSX_ARCHITECTURES=arm64 \ -DCMAKE_MACOSX_RPATH=ON \ -DCMAKE_INSTALL_RPATH="$ICU_ROOT/lib" \ $CMAKE_EXTRA_ARGS From 64b2fdf4b8855110642e6bd42e4f574929ad7c1a Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 18:57:24 +0100 Subject: [PATCH 17/19] . --- .github/workflows/ci.yml | 12 +++--------- bindings/python/setup.py | 10 ++++------ .../python/tools/prepare_build_environment_macos.sh | 3 +-- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1bd290..851d1c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,22 +144,16 @@ jobs: DYLD_LIBRARY_PATH=${GITHUB_WORKSPACE}/icu/lib:${DYLD_LIBRARY_PATH} CIBW_REPAIR_WHEEL_COMMAND_MACOS: | set -e - - echo "=== Starting wheel repair ===" - echo "Wheel: {wheel}" - echo "Destination: {dest_dir}" - echo "ICU path: ${ICU_ROOT}/lib" - - # Use delocate to bundle ICU dylibs into the wheel + echo "=== Bundling ICU into wheel ===" delocate-wheel -v -w {dest_dir} {wheel} - echo "=== Inspect repaired wheel ===" REPAIRED_WHEEL=$(ls {dest_dir}/*.whl) + echo "=== Inspecting repaired wheel ===" unzip -q "$REPAIRED_WHEEL" -d /tmp/wheel_check SO_FILE=$(find /tmp/wheel_check -name "_ext*.so" | head -n1) otool -L "$SO_FILE" rm -rf /tmp/wheel_check - echo "=== Wheel repair complete: $REPAIRED_WHEEL ===" + echo "=== Wheel ready: $REPAIRED_WHEEL ===" diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 0fd1aa0..3d0f124 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -51,13 +51,11 @@ def _maybe_add_library_root(lib_name, header_only=False): if icu_lib_dir: icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] libraries.extend(icu_libs) - + for lib in icu_libs: + ldflags.append(f"-l{lib}") # Ensure _ext uses relative rpath to wheel ICU - ldflags.append(f"-Wl,-rpath,@loader_path/../icu/lib") - # Force linking against ICU libraries in ICU_ROOT - ldflags.extend( - [f"-L{icu_lib_dir}", "-licuuc", "-licudata", "-licui18n", "-licuio"] - ) + ldflags.append("-Wl,-rpath,@loader_path/../icu/lib") + elif sys.platform == "win32": cflags = ["/std:c++17", "/d2FH4-"] package_data["pyonmttok"] = ["*.dll"] diff --git a/bindings/python/tools/prepare_build_environment_macos.sh b/bindings/python/tools/prepare_build_environment_macos.sh index e030ce4..c96297d 100755 --- a/bindings/python/tools/prepare_build_environment_macos.sh +++ b/bindings/python/tools/prepare_build_environment_macos.sh @@ -8,7 +8,7 @@ CMAKE_EXTRA_ARGS="" mkdir -p "$ICU_ROOT" -# Only copy ICU if not already present +# Copy ICU only if not present if [ ! -d "$ICU_ROOT/lib" ]; then brew install icu4c ICU_PREFIX="$(brew --prefix icu4c)" @@ -23,7 +23,6 @@ fi pip install cmake -# Build Tokenizer as static lib rm -rf "$ROOT_DIR/build" mkdir -p "$ROOT_DIR/build" From ca43bfb59e409f3c411b75d064d9f0ed5a08e809 Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 19:04:35 +0100 Subject: [PATCH 18/19] . --- bindings/python/setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 3d0f124..dc786f1 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -51,9 +51,14 @@ def _maybe_add_library_root(lib_name, header_only=False): if icu_lib_dir: icu_libs = ["icuuc", "icudata", "icui18n", "icuio"] libraries.extend(icu_libs) + + # Link ICU with full paths so delocate can see it for lib in icu_libs: - ldflags.append(f"-l{lib}") - # Ensure _ext uses relative rpath to wheel ICU + full_lib_path = os.path.join(icu_lib_dir, f"lib{lib}.dylib") + if os.path.isfile(full_lib_path): + ldflags.append(full_lib_path) + + # rpath for runtime ldflags.append("-Wl,-rpath,@loader_path/../icu/lib") elif sys.platform == "win32": From beb13e3b09c52b543244e052fc2dec53fabfc57e Mon Sep 17 00:00:00 2001 From: vince62s Date: Mon, 5 Jan 2026 19:13:54 +0100 Subject: [PATCH 19/19] remove macos --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 851d1c5..c37be91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,10 +96,10 @@ jobs: arch: aarch64 # macOS - - os: macos-latest - arch: x86_64 - - os: macos-latest - arch: arm64 + #- os: macos-latest + # arch: x86_64 + #- os: macos-latest + # arch: arm64 # Windows - os: windows-2019