diff --git a/.travis.yml b/.travis.yml index 18ec909e..c1996d01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -28,9 +28,9 @@ services: - docker # required to support multi-stage build addons: - apt: - packages: - - docker-ce + apt: + packages: + - docker-ce before_install: - "./tools/travis/setup.sh" @@ -39,13 +39,28 @@ script: - "./tools/travis/build.sh && ./tools/travis/test.sh" deploy: - provider: script - script: "./tools/travis/publish.sh nimbella 3 ${TRAVIS_TAG} && ./tools/travis/publish.sh nimbella 3-ai ${TRAVIS_TAG} && ./tools/travis/publish.sh nimbella 39 ${TRAVIS_TAG}" + script: "./tools/travis/publish.sh nimbella 3 ${TRAVIS_TAG} && ./tools/travis/publish.sh nimbella 3-ai ${TRAVIS_TAG} && ./tools/travis/publish.sh nimbella 39 ${TRAVIS_TAG} && ./tools/travis/publish.sh nimbella 311 ${TRAVIS_TAG}" on: tags: true all_branches: true repo: nimbella-corp/openwhisk-runtime-python - provider: script - script: "./tools/travis/publish.sh nimbella 3 nightly && ./tools/travis/publish.sh nimbella 3-ai nightly && ./tools/travis/publish.sh nimbella 39 nightly" + script: "./tools/travis/publish.sh nimbella 3 nightly && ./tools/travis/publish.sh nimbella 3-ai nightly && ./tools/travis/publish.sh nimbella 39 nightly && ./tools/travis/publish.sh nimbella 311 nightly" on: branch: dev repo: nimbella-corp/openwhisk-runtime-python +env: + global: + - secure: "N5g9wUkuz5N4KM2KlJLpl/3X1l6d/ZFikquDDibpEyMCbhN6dPSpVu3Ji2M5/fcCd49IUnAZbrD7wlIr/zLdBYOBX+ERsjG7phPaIl23xZtJ/J7wTxCA+5i2uI4Oj1yh81OffLf0dxdx+sfRizKPyRpUqwTI0PgntPJKeqUxtILrnNDQ/yZEvSSiqe9ndYYfpZcYJj+IYUc/iXw8CMg3J2s/dTNXgd4o5ADUkIMIcgE+5RzT6R0ODWeK8DRvoVTstRauPR2NdCRratoCCUyue0aIXO/ZVJsdzFQbtn2YC/pbN3bf6acLVAOt2OKd7X1ulwU2ZSKgDcYVWxVzOEk8dojqHuEi22PdsrFG6sLmm/s4ul3IVL50vYwfLpRhCT7dEHK1cIZPI2kUQD+gQ8+cUCRbJUfmTCPI+lRKLG5orAjxSbvS5P+IcAgRBzzGbyrtlpZucBHvsXwOShcisnd475vQlKGzwnEU/hk2WjL5TRzEqTy3zeEewU1PU+1UVKqAOaXacemH82a4lLf/7PT8wdwuASewv1ROhJy+JGa/wWG4uFTyv+OqrslM0i897sJEpsdq0WjMWEDMjc58xZeZFSY3QH+XWXQ6xB8uKe2iVR3t5hMwKxBnjTFDjmY52Uyo/rIBY3qDSUfaOOqepSu48DCXwkcv0olkTXLhcvg9ZJ4=" + - secure: "ifre4ptsRQX+frr4VzVtEtl0QVDDwS2zp93cGXC9ihE8vYrCS+zoRieJ4vSGqs8p0g35H/1WYFhgBGeJvJXRLaN//CGipYVqavjz2ETfybCptN6j0EDc803CbhZK5rOgiBNhdMk8x2H45dcdp5zNLhtvDU/KzgbktT8DkxY0AzKcVki2BCTteChCGCDMKrVkMwlrGhiXSxvcWltA2jnCGvYt9Niv2dogmjFUMmC8BAYCq0hbRY4gK1u7Gq42HCmxwYq2V/5SMCi4bWXa02CdmXA9IhTKXy8EHEk/gFU7ktyyAqgv5ubf0GmW0Qor+b6EfkUHclP+hNV6d3AZI+IbdRWKrSbjPhOOfmnkJ2oFPTYoDPJv0KPb5qoGrHW2TkjqRTQWucTWBDkd6c1m3BMYjSZa5BcCP2mQ6eY/aqX+4YxBHe4tWviNgHiDHGZ/w9iNkcnJ2X6ga6UJg1jeGM7IHW/vqq7EG1YCXeFva38SGPvoDEWVXXzs5HIiJehTHsIgJnHIC+8wGpk4iHyRDQocJsByhsNzQKHHoL9ylep9dJFa6BTOxEvRW+lIMVYVktkYYOsw1rexRQEO9fHQn9cPFxJfKzyZ9Poibt8flciPNDOD8QB26yDZGoqI9Vy2hYXBPLACX6U/KB45Pp0doAT7IdiatpM7uUSlRJN2R68XASc=" + +notifications: + slack: + secure: ipYx0BX7TwWmG+qeyAllxBxrPn21dJubfEbALn8yTOZfykTWpVtohWrqieAnc6kFQRqz1TAs1wzyjvTcutaIMi9C7FcJrYRIMUMpyID3ca5FeZopr/LBW53XHj/SgZWwFRAnWsjYnSydfZJnUBInI6por8jdVqVgsCMTDsvcXFEoHdVj9M93aB7y8pgqZhfYkQOPojktDu7nMxjsjKJN7dT+g/1kVWtS8DQQCXHTPP44YquxOCgAFn+CZjes9Jm+vz5lk5Azksi48yiHwFiLYZo5gcnwr2n2IspjBs6db10rkCe693vIpoW4T3KujMIKwVXrgUl4pDME1nN/loEBHm0DjKoeOD13DHqITMTPX4sR94TCGf0lMyaag/JOETdDgC9hAbh4aSx1cj4hGXiuGrNMbYblMBsjvujqFp2nraw/DE93pHGRY9lbBZIjEVhj/jBmOv1I3yRdXHj4VybZc4JGnfQVGz/RYDOEmKGoLoVWFp+hGAHDqVos9ZvEBByMAAmgVD2/D5cBER3oskp/JJjKmFLk614FcNh4D7/+bYAFhxbCgnoLHH5GeSo0q73O3JSi5pZggEIN9xis7SVQgPl+MUDTe5N3X3J3bQc7cMYGLzh0VNSyYaHuW2P02vWBNE/FAkL8UGrfGsXnDyGRRxo8fYHTXpIjhIkE7tY5eZQ= + on_pull_requests: false + on_success: change + on_failure: always + webhooks: + urls: + # travis2slack webhook to enable DMs on openwhisk-team.slack.com to PR authors with TravisCI results + secure: "JoCPajRbNQ+8Tpu7XyjtLT/pbbA4vhEpScYs7YIE4+iYtZLNXpqXdX/9+tzX9uGsh/+4DoIVMXjcKjDKglgtffgv8RF1wVblIpC8pIcPVaUxWn1KitjZCgFEQnxpT74DgJng3C1ADHon5htNdZ/vhfXDtV6XJ1/mvtWJZCC0ZsWuvRiwnRwZdIkrZi7Fpl6fqTOOAcy7KV8IeKi+GMInjvPrVdG3uLgQO2tOnoKK4oFvDT1sV3hLX8nx6GYAers98AtbvNyiSyg4wAUFV6HTiF7Kc4rHWqPgu8aQRfO8yTlZ4XomAHR1HcujANgoG1lvij28N7pkgRvjUBpPP82qYKlx4xC9RxeH40MjlYiCLHWUswyFb9QGRsnbGtCamME3kBiC3jO86avvD16PWwCbA1q/Y7bLhUZUCUOGHzxvb3n2kCgkcjcbwIiS8+/2aJsIJ8TE7Y6sjrMHJEGILO57gY/JdCrevn4MOfnyCiqPgWdnPTUzcXcYoH/hZctUsOISc2M5yZsNAVHyHFZAbDDdgCLAVi9SAIRhB9ipn/JbpnHzPhu7eW46T6cDMdBAw0FUPvrGch20qhABILU7cFCDZLJ/DouKGAOn1XPXo5FUBjyM854jjm2jVs6IpOdUecW1lGdafB3d0AJXMm4vBXU8VmCuvElurnoE+FNpKZpmjms=" diff --git a/build.gradle b/build.gradle index 4ebd4b8c..efdfee41 100644 --- a/build.gradle +++ b/build.gradle @@ -17,14 +17,17 @@ buildscript { repositories { - jcenter() + mavenCentral() + gradlePluginPortal() } dependencies { - classpath "cz.alenkacz:gradle-scalafmt:${gradle.scalafmt.version}" + classpath "cz.alenkacz.gradle.scalafmt:cz.alenkacz.gradle.scalafmt.gradle.plugin:${gradle.scalafmt.version}" } } subprojects { - apply plugin: 'scalafmt' - scalafmt.configFilePath = gradle.scalafmt.config + plugins.withId('scala') { + apply plugin: 'cz.alenkacz.gradle.scalafmt' + scalafmt.configFilePath = gradle.scalafmt.config + } } diff --git a/core/python311Action/Dockerfile b/core/python311Action/Dockerfile index 0d1bffd4..02ac65dd 100644 --- a/core/python311Action/Dockerfile +++ b/core/python311Action/Dockerfile @@ -15,50 +15,92 @@ # limitations under the License. # -# build go proxy from source +# STAGE 1: Build Go Proxy from source ARG GO_PROXY_BASE_IMAGE=golang:1.20 -FROM $GO_PROXY_BASE_IMAGE AS builder -ARG GO_PROXY_GITHUB_USER=nimbella-corp -ARG GO_PROXY_GITHUB_BRANCH=dev +FROM $GO_PROXY_BASE_IMAGE AS proxy-builder +ARG GO_PROXY_GITHUB_USER=nimbella +ARG GO_PROXY_GITHUB_BRANCH=master RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src \ && cd /src \ && env GO111MODULE=on CGO_ENABLED=0 go build -o /bin/proxy main/proxy.go -FROM python:3.11-slim-bullseye +# STAGE 2: Build Python Wheels (Heavy Build Environment) +# This stage installs Rust, Bun, and C compilers needed only for building Daft. +FROM python:3.11-slim-bullseye AS python-builder + +# Install system dependencies for building native extensions +RUN apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ + git \ + curl \ + build-essential \ + python3-dev \ + libssl-dev \ + pkg-config \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust toolchain (required for Daft engine) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install Bun (required specifically for Daft Dashboard release builds) +RUN curl -fsSL https://bun.sh/install | bash +ENV PATH="/root/.bun/bin:${PATH}" + +WORKDIR /build +COPY requirements.txt . + +# MODIFIED: Install build-time dependencies and use --no-build-isolation. +# Disabling build isolation forces pip to use the Rust/Bun toolchains we installed above, +# cutting hours off the build time and preventing redundant toolchain downloads. +RUN pip install --no-cache-dir maturin pyarrow>=8.0.0 \ + && pip wheel --no-cache-dir --no-build-isolation --wheel-dir /wheels git+https://github.com/Eventual-Inc/Daft.git@v0.4.8 -# select the builder to use +# STAGE 3: Final Production Image (Slim) +FROM python:3.11-slim-bullseye ARG GO_PROXY_BUILD_FROM=release -# Install common modules for python +# Runtime dependencies only +RUN apt-get -o Acquire::Retries=3 update && apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy built artifacts from earlier stages +COPY --from=proxy-builder /bin/proxy /bin/proxy +COPY --from=python-builder /wheels /wheels + +# Install the pre-compiled Daft wheel +RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels + +# MODIFIED: Filter requirements.txt to remove the 'daft' Git line. +# This prevents pip from looking for 'git' in this slim stage, +# as daft is already provided by the wheel installed in the previous step. COPY requirements.txt requirements.txt -RUN pip install --no-cache-dir -r requirements.txt +RUN grep -v "daft @" requirements.txt > requirements_slim.txt \ + && pip install --no-cache-dir -r requirements_slim.txt \ + && rm requirements_slim.txt -# install the functions-deployer +# Install the functions-deployer ARG DEPLOYER_DOWNLOAD SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl \ - && curl -L ${DEPLOYER_DOWNLOAD} | tar xzf - \ +RUN curl -L ${DEPLOYER_DOWNLOAD} | tar xzf - \ && rm -fr /usr/local/lib/dosls && mv dosls /usr/local/lib \ - && rm -f /usr/local/bin/dosls && ln -s /usr/local/lib/dosls/bootstrap /usr/local/bin/dosls \ - && rm -rf /var/lib/apt/lists/* + && rm -f /usr/local/bin/dosls && ln -s /usr/local/lib/dosls/bootstrap /usr/local/bin/dosls +# Runtime ActionLoop environment setup RUN mkdir -p /action WORKDIR / COPY bin/compile /bin/compile COPY lib/launcher.py /lib/launcher.py COPY lib/prelauncher.py /lib/prelauncher.py +# OpenWhisk standard environment variables # log initialization errors ENV OW_LOG_INIT_ERROR=1 # the launcher must wait for an ack ENV OW_WAIT_FOR_ACK=1 -# using the runtime name to identify the execution environment -#ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.9 # compiler script ENV OW_COMPILER=/bin/compile - ENV OW_INIT_IN_ACTIONLOOP=/lib/prelauncher.py -COPY --from=builder /bin/proxy /bin/proxy ENTRYPOINT ["/bin/proxy"] diff --git a/gradle/docker.gradle b/gradle/docker.gradle index 790c1e10..451f9714 100644 --- a/gradle/docker.gradle +++ b/gradle/docker.gradle @@ -54,8 +54,16 @@ if(project.hasProperty('dockerHost')) { } if(project.hasProperty('dockerBuildArgs')) { - dockerBuildArgs.split(' ').each { arg -> - dockerBuildArg += ['--build-arg', arg] + // Split the string into a list of individual arguments + def argsList = project.dockerBuildArgs.split(' ') + argsList.each { arg -> + if (arg.startsWith('--')) { + // It's a flag (like --no-cache), add it directly + dockerBuildArg += [arg] + } else { + // It's a variable (like KEY=VAL), add the prefix + dockerBuildArg += ['--build-arg', arg] + } } } diff --git a/settings.gradle b/settings.gradle index c21b676c..5e6a8118 100644 --- a/settings.gradle +++ b/settings.gradle @@ -36,7 +36,7 @@ gradle.ext.scala = [ ] gradle.ext.scalafmt = [ - version: '1.5.0', + version: '1.16.2', config: new File(rootProject.projectDir, '.scalafmt.conf') ]