v1.38.1

.github/workflows/ci.yml

@@ -121,12 +121,14 @@ jobs:
           output-dir: wheelhouse
         env:
           # ---- Build selection ----
+          # Linux/Windows: Python 3.10, 3.11, 3.12
+          # macOS: Python 3.10, 3.11 only (skip 3.12)
           CIBW_BUILD: "cp310-* cp311-* cp312-*"
-          CIBW_SKIP: "pp* *-musllinux_*"
+          CIBW_SKIP: "pp* *-musllinux_* cp312-macosx*"
           CIBW_ARCHS: ${{ matrix.arch }}
 
           # ---- Dependencies ----
-          CIBW_BEFORE_BUILD: pip install pybind11>=2.10.4
+          CIBW_BEFORE_BUILD: pip install pybind11==2.10.1
 
           # ---- Linux ----
           CIBW_BEFORE_ALL_LINUX: bindings/python/tools/prepare_build_environment_linux.sh
@@ -138,24 +140,8 @@ jobs:
 
           # ---- macOS ----
           CIBW_BEFORE_ALL_MACOS: bindings/python/tools/prepare_build_environment_macos.sh
-          CIBW_ENVIRONMENT_MACOS: |
-            TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install
-            ICU_ROOT=${GITHUB_WORKSPACE}/icu
-            DYLD_LIBRARY_PATH=${GITHUB_WORKSPACE}/icu/lib:${DYLD_LIBRARY_PATH}
-          CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
-            set -e
-            echo "=== Bundling ICU into wheel ==="
-            delocate-wheel -v -w {dest_dir} {wheel}
-
-            REPAIRED_WHEEL=$(ls {dest_dir}/*.whl)
-            echo "=== Inspecting repaired wheel ==="
-            unzip -q "$REPAIRED_WHEEL" -d /tmp/wheel_check
-            SO_FILE=$(find /tmp/wheel_check -name "_ext*.so" | head -n1)
-            otool -L "$SO_FILE"
-            rm -rf /tmp/wheel_check
-            echo "=== Wheel ready: $REPAIRED_WHEEL ==="
-
-
+          CIBW_ENVIRONMENT_MACOS: TOKENIZER_ROOT=${GITHUB_WORKSPACE}/build/install
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
 
           # ---- Windows ----
           CIBW_BEFORE_ALL_WINDOWS: bash bindings/python/tools/prepare_build_environment_windows.sh
@@ -164,6 +150,7 @@ jobs:
           # ---- Tests ----
           CIBW_TEST_COMMAND: pytest {project}/bindings/python/test/test.py
           CIBW_TEST_REQUIRES: pytest
+          CIBW_TEST_SKIP: "*-macosx*"
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4

CHANGELOG.md

@@ -26,6 +26,12 @@ The project follows [semantic versioning 2.0.0](https://semver.org/). The API co
 
 ### Fixes and improvements
 
+## [v1.38.1](https://github.com/OpenNMT/Tokenizer/releases/tag/v1.38.1) (2026-01-09)
+
+### Fixes and improvements
+
+* python 3.12 was not uploaded in 1.38.0
+
 ## [v1.38.0](https://github.com/OpenNMT/Tokenizer/releases/tag/v1.38.0) (2025-12-30)
 
 ### Fixes and improvements

bindings/python/pyonmttok/version.py

@@ -1,3 +1,3 @@
 """Version information."""
 
-__version__ = "1.38.0"
+__version__ = "1.38.1"

bindings/python/setup.py

@@ -40,27 +40,14 @@ def _maybe_add_library_root(lib_name, header_only=False):
 
 
 _maybe_add_library_root("TOKENIZER")
-icu_lib_dir = _maybe_add_library_root("ICU")
 
 cflags = ["-std=c++17", "-fvisibility=hidden"]
 ldflags = []
 package_data = {}
 
 if sys.platform == "darwin":
     cflags.append("-mmacosx-version-min=10.14")
-    if icu_lib_dir:
-        icu_libs = ["icuuc", "icudata", "icui18n", "icuio"]
-        libraries.extend(icu_libs)
-
-        # Link ICU with full paths so delocate can see it
-        for lib in icu_libs:
-            full_lib_path = os.path.join(icu_lib_dir, f"lib{lib}.dylib")
-            if os.path.isfile(full_lib_path):
-                ldflags.append(full_lib_path)
-
-        # rpath for runtime
-        ldflags.append("-Wl,-rpath,@loader_path/../icu/lib")
-
+    ldflags.append("-Wl,-rpath,/usr/local/lib")
 elif sys.platform == "win32":
     cflags = ["/std:c++17", "/d2FH4-"]
     package_data["pyonmttok"] = ["*.dll"]
@@ -72,7 +59,7 @@ def _maybe_add_library_root(lib_name, header_only=False):
     extra_link_args=ldflags,
     include_dirs=include_dirs,
     library_dirs=library_dirs,
-    libraries=libraries,
+    libraries=["OpenNMTTokenizer"],
 )
 
 setup(
@@ -108,5 +95,7 @@ def _maybe_add_library_root(lib_name, header_only=False):
     packages=find_packages(),
     package_data=package_data,
     python_requires=">=3.10",
+    setup_requires=["pytest-runner"],
+    tests_require=["pytest"],
     ext_modules=[tokenizer_module],
 )

bindings/python/tools/prepare_build_environment_macos.sh

@@ -15,26 +15,26 @@ if [ ! -d "$ICU_ROOT/lib" ]; then
     rsync -a "$ICU_PREFIX/" "$ICU_ROOT/"
 fi
 
-export DYLD_LIBRARY_PATH="$ICU_ROOT/lib:$DYLD_LIBRARY_PATH"
+# Remove dynamic libraries to force static linking
+rm -f "$ICU_ROOT/lib/"*.dylib || true
 
 if [[ "$(uname -m)" == "arm64" ]]; then
     CMAKE_EXTRA_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64"
 fi
 
 pip install cmake
 
-rm -rf "$ROOT_DIR/build"
-mkdir -p "$ROOT_DIR/build"
+# Build Tokenizer
+rm -rf build
+mkdir build
+cd build
 
 cmake \
-  -S "$ROOT_DIR" \
-  -B "$ROOT_DIR/build" \
   -DLIB_ONLY=ON \
-  -DBUILD_SHARED_LIBS=ON \
   -DICU_ROOT="$ICU_ROOT" \
   -DCMAKE_INSTALL_PREFIX="$ROOT_DIR/build/install" \
-  -DCMAKE_INSTALL_RPATH="$ICU_ROOT/lib" \
-  $CMAKE_EXTRA_ARGS
+  $CMAKE_EXTRA_ARGS \
+  ..
 
-cmake --build "$ROOT_DIR/build" --target install -j2
-export DYLD_LIBRARY_PATH="$ROOT_DIR/build/install/lib:$DYLD_LIBRARY_PATH"
+VERBOSE=1 make -j2 install
+cd "$ROOT_DIR"