From 498372a5d16c3ed5fb4f3b85265cff097ffe72e4 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Fri, 19 Dec 2025 23:27:47 +0530 Subject: [PATCH 1/4] feat: Replace Docker builds with conda-based CI workflows Add conda/micromamba-based GitHub Actions workflows for building and testing ml-metadata wheels. Uses GCC 8.5.0 to match manylinux2014 compatibility, supports Python 3.9-3.11, and includes automated PyPI publishing. --- .bazelrc | 1 + .github/workflows/conda-build.yml | 97 +++++++++++++++++++++++++++++++ .github/workflows/conda-test.yml | 69 ++++++++++++++++++++++ ci/environment.yml | 22 +++++++ 4 files changed, 189 insertions(+) create mode 100644 .github/workflows/conda-build.yml create mode 100644 .github/workflows/conda-test.yml create mode 100644 ci/environment.yml diff --git a/.bazelrc b/.bazelrc index acfc7bdda..90e765307 100644 --- a/.bazelrc +++ b/.bazelrc @@ -23,5 +23,6 @@ build --protocopt=--experimental_allow_proto3_optional # parameter 'user_link_flags' is deprecated and will be removed soon. # It may be temporarily re-enabled by setting --incompatible_require_linker_input_cc_api=false build --incompatible_require_linker_input_cc_api=false + build:macos --apple_platform_type=macos build:macos_arm64 --cpu=darwin_arm64 diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml new file mode 100644 index 000000000..ea35e03d0 --- /dev/null +++ b/.github/workflows/conda-build.yml @@ -0,0 +1,97 @@ +name: Build ml-metadata with Conda + +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.9", "3.10", "3.11"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ci/environment.yml + cache-environment: true + create-args: >- + python=${{ matrix.python-version }} + + - name: Display environment info + shell: bash -l {0} + run: | + micromamba info + micromamba list + + - name: Install Bazel + shell: bash -l {0} + run: | + # Install Bazelisk (manages Bazel versions) + if [ "$RUNNER_OS" == "Linux" ]; then + curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64 + elif [ "$RUNNER_OS" == "macOS" ]; then + curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin-amd64 + fi + chmod +x /tmp/bazelisk + sudo mv /tmp/bazelisk /usr/local/bin/bazel + echo "USE_BAZEL_VERSION=6.1.0" >> $GITHUB_ENV + bazel --version + + - name: Build the package + shell: bash -l {0} + run: | + python setup.py bdist_wheel + + - name: Repair wheel (Linux) + if: runner.os == 'Linux' + shell: bash -l {0} + run: | + WHEEL_PATH="$(ls dist/*.whl)" + WHEEL_DIR=$(dirname "${WHEEL_PATH}") + auditwheel repair --plat manylinux2014_x86_64 -w "${WHEEL_DIR}" "${WHEEL_PATH}" + rm "${WHEEL_PATH}" + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4.4.0 + with: + name: ml-metadata-wheel-${{ matrix.os }}-py${{ matrix.python-version }} + path: dist/*.whl + + upload_to_pypi: + name: Upload to PyPI + runs-on: ubuntu-latest + if: (github.event_name == 'release' && startsWith(github.ref, 'refs/tags')) || (github.event_name == 'workflow_dispatch') + needs: [build] + environment: + name: pypi + url: https://pypi.org/p/ml-metadata/ + permissions: + id-token: write + steps: + - name: Retrieve wheels + uses: actions/download-artifact@v4.1.8 + with: + merge-multiple: true + path: wheels + + - name: List the build artifacts + run: | + ls -lAs wheels/ + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1.9 + with: + packages_dir: wheels/ diff --git a/.github/workflows/conda-test.yml b/.github/workflows/conda-test.yml new file mode 100644 index 000000000..77ec6d543 --- /dev/null +++ b/.github/workflows/conda-test.yml @@ -0,0 +1,69 @@ +name: Test ml-metadata with Conda + +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.9", "3.10", "3.11"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ci/environment.yml + cache-environment: true + create-args: >- + python=${{ matrix.python-version }} + + - name: Display environment info + shell: bash -l {0} + run: | + micromamba info + micromamba list + + - name: Install Bazel + shell: bash -l {0} + run: | + # Install Bazelisk (manages Bazel versions) + if [ "$RUNNER_OS" == "Linux" ]; then + curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64 + elif [ "$RUNNER_OS" == "macOS" ]; then + curl -Lo /tmp/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin-amd64 + fi + chmod +x /tmp/bazelisk + sudo mv /tmp/bazelisk /usr/local/bin/bazel + echo "USE_BAZEL_VERSION=6.1.0" >> $GITHUB_ENV + bazel --version + + - name: Build the package + shell: bash -l {0} + run: | + python setup.py bdist_wheel + + - name: Install built wheel (Linux/macOS) + shell: bash -l {0} + run: | + pip install dist/*.whl + + - name: Run tests + shell: bash -l {0} + run: | + # cleanup (interferes with tests) + rm -rf bazel-* + # run tests + pytest -vv diff --git a/ci/environment.yml b/ci/environment.yml new file mode 100644 index 000000000..113408e3f --- /dev/null +++ b/ci/environment.yml @@ -0,0 +1,22 @@ +# Conda environment for building and testing ml-metadata on Linux +name: mlmd-dev +channels: + - conda-forge +dependencies: + # Note: Bazel is installed separately via official installer (conda package is unreliable) + - setuptools + - wheel + - pip + - numpy + - pytest + - pytest-cov + - patchelf # For wheel repair on Linux + - cmake=3.29 + + # C/C++ compilers - GCC 8.x to match manylinux2014 devtoolset-8 + - gcc_linux-64=8.5.0 + - gxx_linux-64=8.5.0 + - sysroot_linux-64=2.17 # CentOS 7/manylinux2014 compatible glibc headers + + - pip: + - auditwheel # For manylinux wheel compliance From df9d7e36d5b05e290e01d628164fb1884fa02cb5 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Thu, 8 Jan 2026 22:59:35 +0530 Subject: [PATCH 2/4] build(ci): bump Bazel to 6.5.0 and update manylinux toolchain - Bazel: Raise minimum to 6.5.0 and update repo version - CI: Use Bazel 6.5.0 in conda-build/test GitHub Actions - Docker: Switch manylinux image to bazel-6.5.0 tag - Toolchain: Use devtoolset-10 for wheel builds - Scope: Build/CI-only; no MLMD runtime or API changes Motivation: align CI/local builds with Bazel 6.5.0 for stability and modernize the wheel build toolchain. --- .github/workflows/conda-build.yml | 2 +- .github/workflows/conda-test.yml | 2 +- WORKSPACE | 2 +- ml_metadata/.bazelversion | 2 +- ml_metadata/tools/docker_build/Dockerfile.manylinux2010 | 2 +- ml_metadata/tools/docker_build/build_manylinux.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index ea35e03d0..4ab6f93e1 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -47,7 +47,7 @@ jobs: fi chmod +x /tmp/bazelisk sudo mv /tmp/bazelisk /usr/local/bin/bazel - echo "USE_BAZEL_VERSION=6.1.0" >> $GITHUB_ENV + echo "USE_BAZEL_VERSION=6.5.0" >> $GITHUB_ENV bazel --version - name: Build the package diff --git a/.github/workflows/conda-test.yml b/.github/workflows/conda-test.yml index 77ec6d543..d945b8388 100644 --- a/.github/workflows/conda-test.yml +++ b/.github/workflows/conda-test.yml @@ -47,7 +47,7 @@ jobs: fi chmod +x /tmp/bazelisk sudo mv /tmp/bazelisk /usr/local/bin/bazel - echo "USE_BAZEL_VERSION=6.1.0" >> $GITHUB_ENV + echo "USE_BAZEL_VERSION=6.5.0" >> $GITHUB_ENV bazel --version - name: Build the package diff --git a/WORKSPACE b/WORKSPACE index 246515185..313a86c3e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -282,4 +282,4 @@ ml_metadata_workspace() # Specify the minimum required bazel version. load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check("6.1.0") +versions.check("6.5.0") diff --git a/ml_metadata/.bazelversion b/ml_metadata/.bazelversion index 44f2af0f6..f92706dd9 100644 --- a/ml_metadata/.bazelversion +++ b/ml_metadata/.bazelversion @@ -11,4 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -6.1.0 +6.5.0 diff --git a/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 b/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 index 77a5174cb..5c2ca1673 100644 --- a/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 +++ b/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 @@ -15,6 +15,6 @@ # Dockerfile for building a manylinux2010 MLMD wheel. # This docker image is essentially pypa/manylinux2010 + bazel. -FROM gcr.io/tfx-oss-public/manylinux2014-bazel:bazel-6.1.0 +FROM gcr.io/tfx-oss-public/manylinux2014-bazel:bazel-6.5.0 WORKDIR /build CMD ["ml_metadata/tools/docker_build/build_manylinux.sh"] diff --git a/ml_metadata/tools/docker_build/build_manylinux.sh b/ml_metadata/tools/docker_build/build_manylinux.sh index 0fb17c459..05944d5e9 100755 --- a/ml_metadata/tools/docker_build/build_manylinux.sh +++ b/ml_metadata/tools/docker_build/build_manylinux.sh @@ -25,7 +25,7 @@ WORKING_DIR=$PWD function setup_environment() { - source scl_source enable devtoolset-8 + source scl_source enable devtoolset-10 source scl_source enable rh-python38 if [[ -z "${PYTHON_VERSION}" ]]; then echo "Must set PYTHON_VERSION env to 39|310|311|"; exit 1; From 209becd983419e4bd52f5675ee03e7df930538e0 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Fri, 16 Jan 2026 17:15:46 +0530 Subject: [PATCH 3/4] fix: Pin CMake to 3.24.4 in Docker build for libmysqlclient compatibility CMake 3.29+ removed support for CMake < 3.5 syntax, breaking libmysqlclient (mariadb-connector-c v3.0.8) builds. Pin to CMake 3.24.4 to maintain backward compatibility while supporting current toolchains. --- ml_metadata/tools/docker_build/Dockerfile.manylinux2010 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 b/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 index 5c2ca1673..a2cfbaafc 100644 --- a/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 +++ b/ml_metadata/tools/docker_build/Dockerfile.manylinux2010 @@ -16,5 +16,13 @@ # This docker image is essentially pypa/manylinux2010 + bazel. FROM gcr.io/tfx-oss-public/manylinux2014-bazel:bazel-6.5.0 + +# Install CMake 3.24 for compatibility with older CMake projects (libmysqlclient) +RUN yum remove -y cmake cmake3 && \ + curl -L https://github.com/Kitware/CMake/releases/download/v3.24.4/cmake-3.24.4-linux-x86_64.tar.gz | tar -xz -C /opt && \ + ln -sf /opt/cmake-3.24.4-linux-x86_64/bin/cmake /usr/local/bin/cmake && \ + ln -sf /opt/cmake-3.24.4-linux-x86_64/bin/ctest /usr/local/bin/ctest && \ + ln -sf /opt/cmake-3.24.4-linux-x86_64/bin/cpack /usr/local/bin/cpack + WORKDIR /build CMD ["ml_metadata/tools/docker_build/build_manylinux.sh"] From 918efee1d3d38fffb0cb6a33084ea00c7a989d17 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Fri, 16 Jan 2026 20:56:17 +0530 Subject: [PATCH 4/4] chore: Remove duplicate upload_to_pypi job from build.yml The upload_to_pypi job is already present in conda-build.yml, which will supersede the Docker-based build.yml workflow. --- .github/workflows/build.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e68283ecc..70a637141 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,29 +25,3 @@ jobs: uses: ./.github/reusable-build with: python-version: ${{ matrix.python-version }} - - upload_to_pypi: - name: Upload to PyPI - runs-on: ubuntu-latest - if: (github.event_name == 'release' && startsWith(github.ref, 'refs/tags')) || (github.event_name == 'workflow_dispatch') - needs: [build] - environment: - name: pypi - url: https://pypi.org/p/ml-metadata/ - permissions: - id-token: write - steps: - - name: Retrieve wheels - uses: actions/download-artifact@v4.1.8 - with: - merge-multiple: true - path: wheels - - - name: List the build artifacts - run: | - ls -lAs wheels/ - - - name: Upload to PyPI - uses: pypa/gh-action-pypi-publish@release/v1.9 - with: - packages_dir: wheels/