diff --git a/.travis.yml b/.travis.yml index d2508dff..739287de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,29 +39,13 @@ script: - "./tools/travis/build.sh && ./tools/travis/test.sh" deploy: - provider: script - script: "./tools/travis/publish.sh openwhisk ${TRAVIS_TAG%@*} ${TRAVIS_TAG##*@}" + script: "./tools/travis/publish.sh nimbella 3 ${TRAVIS_TAG} && ./tools/travis/publish.sh nimbella 3-ai ${TRAVIS_TAG} && ./tools/travis/publish.sh nimbella 39 ${TRAVIS_TAG} **&& ./tools/travis/publish.sh nimbella 311 ${TRAVIS_TAG}**" on: tags: true all_branches: true - repo: apache/openwhisk-runtime-python + repo: nimbella-corp/openwhisk-runtime-python - provider: script - script: "./tools/travis/publish.sh openwhisk 3 nightly && ./tools/travis/publish.sh openwhisk 3-ai nightly" + script: "./tools/travis/publish.sh nimbella 3 nightly && ./tools/travis/publish.sh nimbella 3-ai nightly && ./tools/travis/publish.sh nimbella 39 nightly **&& ./tools/travis/publish.sh nimbella 311 nightly**" on: - branch: master - repo: apache/openwhisk-runtime-python -env: - global: - - secure: Ac7MlWUpsLwmbcjCigDf+vftiXAEJbf78FVm2QCd4IeFqlRiNbQm3bWRXOOW3Zqh4R4o/wSFpFq0qKPrI9WJjOcxD0GxFboq2iGWTegddfDbENV/IBmjurl6NzYY6dUB7GUSmHYy6tdKnxMhjHzZCn2m4OVWltcMtnOV0G+uYvbnre7vXfb67duzJF+mYpJJ7rC0wGh+N50bYvm0BTtPcQMYJibq6sDTY8Lr1b74QGdM/sODre69CbuiwAA7Ed0hajEcMkv8YnwY0De/PL7hthUyCWSD+i1AhtD+3odA64yX/pAbBzfCcqPoI0Nt5ZmAAUKLJ9pAHCc74dlmYzj9wj1z5TS0nLOE2l++vNvgsA3/c+lqrBv6hc9FmSKx4yfNdFCakVLEfdRJw/45ibIRN5dCW5tDdIx8aciBV8c0CyYgSLRvtVuvw5IX7fbwhHOZ8ftTRzCHS0Nd9zs4x3za+1aucWFQEyrkPtXZIiqeuV7yI4S7yYhGeJy/3KmuN7ljDiTElZb/W9W22ryw/UwCyyZBGnvhomAwInS3E51Jv8GuOU9FVliGzGs2yLuVQ1FjIZ25Au5W4TLTor/jE+dxOlH4u25neqsKBxzG4iyJs3AfJnycD8op7JbTG64OqpVvaB8A4N3P416IxGALSFVv2BWSLY4qWBRVKHbZAHn2tTg= - - secure: fte97QoQReAM0obk/tjxYvF/zhRbcyIFxa1jnFuv1m1YKA/pEWue5uMIDZZa1pynN9rFFgOvR2W34Gm2aHl0wJs18Oqi2Es1cSfDDmuYguEIYZmlfbrjXY2gvitH5+6JA6fc4aMmrgOav+WVbzdzBqy511CWEDsejWflAnFpKJqTnYdLqe8L8FAJfteY/cj5Bvz7G5o8WbH+ag2CopjkEKkx4KXsJ7DadKI+BbI9hkMUvPVnTULaf7Q5z3oG92g1XwjKrfN6Z2y8tQukWg+n5pfVY0yLpWvd/Q/9HyilB0s06EzORzE00DTLnvYNGaVsx99m6H11e0Tn5Sx5S6rJOmJa5T2vTPGmsojpgHm1Lz12AYVxRmWvFbTLg8KDNN9H6uTc8DKPGb/mLkkD5ruWXJWjvpd+m+eRcpJOZ9cvPXWFYvCF0VAPCbK0DakInnmCvZyQ6ZLFgGrtkNt51kkTZGR68d0UW9rwrBS/JaaRdKIbSOHB346trSO2raYIQhaaAtv+tHCOVVAuoVC5Erzw+bojrWtTNQl9FZ2NM2jFaoeiRmi/rB9hqs8iK2BD20EnWfaMRf5XOkBJXYDQ804EnaOAHP/ZO94Ziojotf5SaXSeWG/zsxbT13u+zTdEf3auDofe8exzCpXuOHAQrB0b+Dy2OSKRPwzMHsfto6/OU9w= -notifications: - slack: - secure: ipYx0BX7TwWmG+qeyAllxBxrPn21dJubfEbALn8yTOZfykTWpVtohWrqieAnc6kFQRqz1TAs1wzyjvTcutaIMi9C7FcJrYRIMUMpyID3ca5FeZopr/LBW53XHj/SgZWwFRAnWsjYnSydfZJnUBInI6por8jdVqVgsCMTDsvcXFEoHdVj9M93aB7y8pgqZhfYkQOPojktDu7nMxjsjKJN7dT+g/1kVWtS8DQQCXHTPP44YquxOCgAFn+CZjes9Jm+vz5lk5Azksi48yiHwFiLYZo5gcnwr2n2IspjBs6db10rkCe693vIpoW4T3KujMIKwVXrgUl4pDME1nN/loEBHm0DjKoeOD13DHqITMTPX4sR94TCGf0lMyaag/JOETdDgC9hAbh4aSx1cj4hGXiuGrNMbYblMBsjvujqFp2nraw/DE93pHGRY9lbBZIjEVhj/jBmOv1I3yRdXHj4VybZc4JGnfQVGz/RYDOEmKGoLoVWFp+hGAHDqVos9ZvEBByMAAmgVD2/D5cBER3oskp/JJjKmFLk614FcNh4D7/+bYAFhxbCgnoLHH5GeSo0q73O3JSi5pZggEIN9xis7SVQgPl+MUDTe5N3X3J3bQc7cMYGLzh0VNSyYaHuW2P02vWBNE/FAkL8UGrfGsXnDyGRRxo8fYHTXpIjhIkE7tY5eZQ= - on_pull_requests: false - on_success: change - on_failure: always - webhooks: - urls: - # travis2slack webhook to enable DMs on openwhisk-team.slack.com to PR authors with TravisCI results - secure: "JoCPajRbNQ+8Tpu7XyjtLT/pbbA4vhEpScYs7YIE4+iYtZLNXpqXdX/9+tzX9uGsh/+4DoIVMXjcKjDKglgtffgv8RF1wVblIpC8pIcPVaUxWn1KitjZCgFEQnxpT74DgJng3C1ADHon5htNdZ/vhfXDtV6XJ1/mvtWJZCC0ZsWuvRiwnRwZdIkrZi7Fpl6fqTOOAcy7KV8IeKi+GMInjvPrVdG3uLgQO2tOnoKK4oFvDT1sV3hLX8nx6GYAers98AtbvNyiSyg4wAUFV6HTiF7Kc4rHWqPgu8aQRfO8yTlZ4XomAHR1HcujANgoG1lvij28N7pkgRvjUBpPP82qYKlx4xC9RxeH40MjlYiCLHWUswyFb9QGRsnbGtCamME3kBiC3jO86avvD16PWwCbA1q/Y7bLhUZUCUOGHzxvb3n2kCgkcjcbwIiS8+/2aJsIJ8TE7Y6sjrMHJEGILO57gY/JdCrevn4MOfnyCiqPgWdnPTUzcXcYoH/hZctUsOISc2M5yZsNAVHyHFZAbDDdgCLAVi9SAIRhB9ipn/JbpnHzPhu7eW46T6cDMdBAw0FUPvrGch20qhABILU7cFCDZLJ/DouKGAOn1XPXo5FUBjyM854jjm2jVs6IpOdUecW1lGdafB3d0AJXMm4vBXU8VmCuvElurnoE+FNpKZpmjms=" - - + branch: dev + repo: nimbella-corp/openwhisk-runtime-python diff --git a/README.md b/README.md index def82e3a..555d436f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ # Apache OpenWhisk runtimes for Python [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0) -[![Build Status](https://travis-ci.com/apache/openwhisk-runtime-python.svg?branch=master)](https://travis-ci.com/apache/openwhisk-runtime-python) +[![Build Status](https://travis-ci.com/nimbella-corp/openwhisk-runtime-python.svg?branch=master)](https://travis-ci.com/nimbella-corp/openwhisk-runtime-python) ## Build Runtimes diff --git a/build.gradle b/build.gradle index 4ebd4b8c..3522adb1 100644 --- a/build.gradle +++ b/build.gradle @@ -15,16 +15,26 @@ * limitations under the License. */ + + buildscript { repositories { - jcenter() + mavenCentral() + gradlePluginPortal() } dependencies { - classpath "cz.alenkacz:gradle-scalafmt:${gradle.scalafmt.version}" + // Dependencies for the build script itself go here } } +plugins { + id "cz.alenkacz.gradle.scalafmt" version "1.16.2" apply false +} + subprojects { - apply plugin: 'scalafmt' - scalafmt.configFilePath = gradle.scalafmt.config + // Only apply if the project is a Scala project or has the property + if (project.name.contains("scala") || project.hasProperty('scalafmt')) { + apply plugin: 'cz.alenkacz.gradle.scalafmt' + scalafmt.configFilePath = gradle.scalafmt.config + } } diff --git a/core/python2ActionLoop/Dockerfile b/core/python2ActionLoop/Dockerfile new file mode 100644 index 00000000..2eeabff3 --- /dev/null +++ b/core/python2ActionLoop/Dockerfile @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# build go proxy from source +FROM golang:1.15 AS builder_source +ARG GO_PROXY_GITHUB_USER=nimbella-corp +ARG GO_PROXY_GITHUB_BRANCH=dev +RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} \ + https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src ;\ + cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \ + mv proxy /bin/proxy + +# or build it from a release +FROM golang:1.15 AS builder_release +ARG GO_PROXY_RELEASE_VERSION=1.15@1.17.0 +RUN curl -sL \ + https://github.com/apache/openwhisk-runtime-go/archive/{$GO_PROXY_RELEASE_VERSION}.tar.gz\ + | tar xzf -\ + && cd openwhisk-runtime-go-*/main\ + && GO111MODULE=on go build -o /bin/proxy + +FROM python:2.7-alpine + +# select the builder to use +ARG GO_PROXY_BUILD_FROM=source + +# Upgrade and install basic Python dependencies +RUN apk add --no-cache \ + bash \ + build-base \ + bzip2-dev \ + curl \ + gcc \ + libc-dev \ + libxslt-dev \ + libxml2-dev \ + libffi-dev \ + linux-headers \ + openssl-dev \ + python-dev + +# Install common modules for python +RUN pip install --no-cache-dir --upgrade pip setuptools six \ + && pip install --no-cache-dir \ + gevent==1.3.6 \ + flask==1.0.2 \ + beautifulsoup4==4.6.3 \ + httplib2==0.11.3 \ + kafka_python==1.4.3 \ + lxml==4.2.5 \ + python-dateutil==2.7.3 \ + requests==2.19.1 \ + scrapy==1.5.1 \ + simplejson==3.16.0 \ + virtualenv==16.0.0 \ + twisted==18.7.0 + +# install nim +ARG NIM_INSTALL_SCRIPT=https://apigcp.nimbella.io/downloads/nim/nim-install-linux.sh +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl ${NIM_INSTALL_SCRIPT} | bash + +RUN mkdir -p /action +WORKDIR / +COPY --from=builder_source /bin/proxy /bin/proxy_source +COPY --from=builder_release /bin/proxy /bin/proxy_release +RUN mv /bin/proxy_${GO_PROXY_BUILD_FROM} /bin/proxy + +ADD bin/compile /bin/compile +ADD lib/launcher.py /lib/launcher.py +# the compiler script +ENV OW_COMPILER=/bin/compile +# log initialization errors +ENV OW_LOG_INIT_ERROR=1 +# the launcher must wait for an ack +ENV OW_WAIT_FOR_ACK=1 +ENTRYPOINT ["/bin/proxy"] diff --git a/core/python2ActionLoop/Makefile b/core/python2ActionLoop/Makefile new file mode 100644 index 00000000..4bae7ae3 --- /dev/null +++ b/core/python2ActionLoop/Makefile @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +IMG=whisk/actionloop-python-v2.7:latest + +build: + docker build -t $(IMG) -f Dockerfile . + +clean: + docker rmi -f $(IMG) + +debug: + docker run -p 8080:8080 \ + -ti --entrypoint=/bin/bash -v $(PWD):/mnt \ + -e OW_COMPILER=/mnt/bin/compile \ + $(IMG) + +.PHONY: build clean debug diff --git a/core/python2ActionLoop/bin/compile b/core/python2ActionLoop/bin/compile new file mode 100755 index 00000000..2b0ea6af --- /dev/null +++ b/core/python2ActionLoop/bin/compile @@ -0,0 +1,115 @@ +#!/usr/bin/env python +"""Python Action Builder +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" + +from __future__ import print_function +import os, os.path, sys, imp, ast, shutil, subprocess, traceback +from os.path import abspath, exists, dirname + +# write a file creating intermediate directories +def write_file(file, body, executable=False): + try: os.makedirs(dirname(file), mode=0o755) + except: pass + with open(file, mode="wb") as f: + f.write(body) + if executable: + os.chmod(file, 0o755) + +# copy a file eventually replacing a substring +def copy_replace(src, dst, match=None, replacement=""): + with open(src, 'rb') as s: + body = s.read() + if match: + body = body.replace(match, replacement) + write_file(dst, body) + +# assemble sources +def sources(launcher, main, src_dir): + # move exec in the right place if exists + src_file = "%s/exec" % src_dir + if exists(src_file): + os.rename(src_file, "%s/__main__.py" % src_dir) + if exists("%s/__main__.py" % src_dir): + os.rename("%s/__main__.py" % src_dir, "%s/main__.py" % src_dir) + + # write the boilerplate in a temp dir + copy_replace(launcher, "%s/exec__.py" % src_dir, + "from main__ import main as main", + "from main__ import %s as main" % main ) + +# compile sources +def build(src_dir, tgt_dir): + # in general, compile your program into an executable format + # for scripting languages, move sources and create a launcher + # move away the action dir and replace with the new + shutil.rmtree(tgt_dir) + shutil.move(src_dir, tgt_dir) + tgt_file = "%s/exec" % tgt_dir + write_file(tgt_file, """#!/bin/bash +if [[ "$__OW_EXECUTION_ENV" == "" || "$(cat $0.env)" == "$__OW_EXECUTION_ENV" ]] +then cd "$(dirname $0)" + exec /usr/local/bin/python exec__.py "$@" +else echo "Execution Environment Mismatch" + echo "Expected: $(cat $0.env)" + echo "Actual: $__OW_EXECUTION_ENV" + exit 1 +fi +""", True) + if os.environ.get("__OW_EXECUTION_ENV"): + write_file("%s.env"%tgt_file, os.environ['__OW_EXECUTION_ENV']) + return tgt_file + +#check if a module exists +def check(tgt_dir, module_name): + try: + + # find module + mod = imp.find_module(module_name, [tgt_dir]) + # parse module + ast.parse(mod[0].read()) + # check virtualenv + path_to_virtualenv = abspath('%s/virtualenv' % tgt_dir) + if os.path.isdir(path_to_virtualenv): + activate_this_file = path_to_virtualenv + '/bin/activate_this.py' + if not os.path.exists(activate_this_file): + # check if this was packaged for windows + activate_this_file = path_to_virtualenv + '/Scripts/activate_this.py' + if not os.path.exists(activate_this_file): + sys.stderr.write("Invalid virtualenv. Zip file does not include 'activate_this.py'.\n") + except ImportError: + sys.stderr.write("Zip file does not include %s\n" % module_name) + except SyntaxError as er: + sys.stderr.write(er.msg) + except Exception as ex: + sys.stderr.write(ex.message) + sys.stderr.flush() + +if __name__ == '__main__': + if len(sys.argv) < 4: + sys.stdout.write("usage: \n") + sys.stdout.flush() + sys.exit(1) + launcher = "%s/lib/launcher.py" % dirname(dirname(sys.argv[0])) + src_dir = abspath(sys.argv[2]) + tgt_dir = abspath(sys.argv[3]) + sources(launcher, sys.argv[1], src_dir) + tgt = build(abspath(sys.argv[2]), tgt_dir) + check(tgt_dir, "main__") + sys.stdout.flush() + sys.stderr.flush() diff --git a/core/python2ActionLoop/build.gradle b/core/python2ActionLoop/build.gradle new file mode 100644 index 00000000..66ecc3aa --- /dev/null +++ b/core/python2ActionLoop/build.gradle @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ext.dockerImageName = 'actionloop-python-v2.7' +apply from: '../../gradle/docker.gradle' diff --git a/core/python2ActionLoop/lib/launcher.py b/core/python2ActionLoop/lib/launcher.py new file mode 100755 index 00000000..e87a81cb --- /dev/null +++ b/core/python2ActionLoop/lib/launcher.py @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function +from os import fdopen +import sys, os, codecs, json, traceback, warnings + +sys.stdout = codecs.getwriter('utf8')(sys.stdout) +sys.stderr = codecs.getwriter('utf8')(sys.stderr) +log_sentinel="XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX\n" + +try: + # if the directory 'virtualenv' is extracted out of a zip file + path_to_virtualenv = os.path.abspath('./virtualenv') + if os.path.isdir(path_to_virtualenv): + # activate the virtualenv using activate_this.py contained in the virtualenv + activate_this_file = path_to_virtualenv + '/bin/activate_this.py' + if not os.path.exists(activate_this_file): # try windows path + activate_this_file = path_to_virtualenv + '/Scripts/activate_this.py' + if os.path.exists(activate_this_file): + with open(activate_this_file) as f: + code = compile(f.read(), activate_this_file, 'exec') + exec(code, dict(__file__=activate_this_file)) + else: + sys.stderr.write("Invalid virtualenv. Zip file does not include 'activate_this.py'.\n") + sys.exit(1) +except Exception: + traceback.print_exc(file=sys.stderr, limit=0) + sys.exit(1) + +# now import the action as process input/output +from main__ import main as main + +env = os.environ +out = fdopen(3, "wb") +if os.getenv("__OW_WAIT_FOR_ACK", "") != "": + out.write(json.dumps({"ok": True}, ensure_ascii=False).encode('utf-8')) + out.write(b'\n') + out.flush() +while True: + line = sys.stdin.readline().decode('utf-8') + if not line: break + args = json.loads(line) + payload = {} + for key in args: + akey = key.encode("ascii", "ignore") + if akey == "value": + payload = args["value"] + else: + env["__OW_%s" % akey.upper()] = args[key].encode("ascii", "ignore") + res = {} + try: + res = main(payload) + except Exception as ex: + print(traceback.format_exc(), file=sys.stderr) + res = {"error": str(ex)} + out.write(json.dumps(res, ensure_ascii=False).encode('utf-8')) + out.write(b'\n') + sys.stdout.write(log_sentinel) + sys.stderr.write(log_sentinel) + sys.stdout.flush() + sys.stderr.flush() + out.flush() diff --git a/core/python311Action/Dockerfile b/core/python311Action/Dockerfile new file mode 100644 index 00000000..06ef0181 --- /dev/null +++ b/core/python311Action/Dockerfile @@ -0,0 +1,86 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# --- BUILDER STAGE: Compile the Go Proxy --- +ARG GO_PROXY_BASE_IMAGE=golang:1.20 +FROM $GO_PROXY_BASE_IMAGE AS builder + +# Use the standard Apache repository and the correct branch ('master') +ARG GO_PROXY_GITHUB_USER=apache +ARG GO_PROXY_GITHUB_BRANCH=master + +RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src \ + && cd /src \ + && env GO111MODULE=on CGO_ENABLED=0 go build -o /bin/proxy main/proxy.go + +# --- RUNTIME STAGE: Python 3.11 --- +FROM python:3.11-slim-bullseye + +# select the builder to use +ARG GO_PROXY_BUILD_FROM=release + +# --- FIX: Install build tools for daft 0.4.8 source build --- +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + gcc \ + build-essential \ + python3-dev \ + curl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# Install Bun +RUN curl -fsSL https://bun.sh/install | bash + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Add Bun and Rust to the PATH +ENV PATH="/root/.bun/bin:/root/.cargo/bin:${PATH}" +# --- END FIX --- + +# Install common modules for python +COPY requirements.txt requirements.txt +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# install the functions-deployer +# ADDED DEFAULT VALUE HERE TO FIX THE CURL ERROR +ARG DEPLOYER_DOWNLOAD=https://do-serverless-tools.nyc3.digitaloceanspaces.com/dosls-5.0.22.tgz +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -L ${DEPLOYER_DOWNLOAD} | tar xzf - \ + && rm -fr /usr/local/lib/dosls && mv dosls /usr/local/lib \ + && rm -f /usr/local/bin/dosls && ln -s /usr/local/lib/dosls/bootstrap /usr/local/bin/dosls + +RUN mkdir -p /action +WORKDIR / +COPY bin/compile /bin/compile +COPY lib/launcher.py /lib/launcher.py +COPY lib/prelauncher.py /lib/prelauncher.py + +# log initialization errors +ENV OW_LOG_INIT_ERROR=1 +# the launcher must wait for an ack +ENV OW_WAIT_FOR_ACK=1 +# compiler script +ENV OW_COMPILER=/bin/compile + +ENV OW_INIT_IN_ACTIONLOOP=/lib/prelauncher.py + +COPY --from=builder /bin/proxy /bin/proxy +ENTRYPOINT ["/bin/proxy"] + diff --git a/core/python311Action/build.gradle b/core/python311Action/build.gradle new file mode 100644 index 00000000..06b82e76 --- /dev/null +++ b/core/python311Action/build.gradle @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ext.dockerImageName = 'action-python-v3.11' +apply from: '../../gradle/docker.gradle' + +distDocker.dependsOn 'copyLib' +distDocker.dependsOn 'copyBin' +distDocker.finalizedBy('cleanup') + +task copyLib(type: Copy) { + from '../python3Action/lib' + into './lib' +} + +task copyBin(type: Copy) { + from '../python3Action/bin' + into './bin' +} + +task cleanup(type: Delete) { + delete 'bin' + delete 'lib' +} diff --git a/core/python36AiAction/Dockerfile b/core/python36AiAction/Dockerfile index bb2dab6d..08d02e1a 100644 --- a/core/python36AiAction/Dockerfile +++ b/core/python36AiAction/Dockerfile @@ -17,8 +17,8 @@ # build go proxy from source FROM golang:1.16 AS builder_source -ARG GO_PROXY_GITHUB_USER=apache -ARG GO_PROXY_GITHUB_BRANCH=master +ARG GO_PROXY_GITHUB_USER=nimbella-corp +ARG GO_PROXY_GITHUB_BRANCH=dev RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} \ https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src ;\ cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \ @@ -52,7 +52,6 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \ vim \ && rm -rf /var/lib/apt/lists/* -# PyTorch # persistent as it fails often RUN while ! pip list | grep torch ;\ do pip install torch ; done ;\ @@ -69,6 +68,11 @@ RUN pip3 install --upgrade pip six wheel &&\ pip3 install --no-cache-dir -r requirements.txt &&\ ln -sf /usr/bin/python3 /usr/local/bin/python +# install nim +ARG NIM_INSTALL_SCRIPT=https://apigcp.nimbella.io/downloads/nim/nim-install-linux.sh +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl ${NIM_INSTALL_SCRIPT} | bash + RUN mkdir -p /action WORKDIR / @@ -84,7 +88,7 @@ ENV OW_LOG_INIT_ERROR=1 # the launcher must wait for an ack ENV OW_WAIT_FOR_ACK=1 # using the runtime name to identify the execution environment -ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.6-ai +#ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.6-ai # compiler script ENV OW_COMPILER=/bin/compile # use utf-8 diff --git a/core/python36AiAction/requirements.txt b/core/python36AiAction/requirements.txt index 44e5a9b7..a66b2f35 100644 --- a/core/python36AiAction/requirements.txt +++ b/core/python36AiAction/requirements.txt @@ -11,6 +11,9 @@ virtualenv == 16.7.9 twisted == 19.10.0 netifaces == 0.10.9 +# Nimbella package +nimbella == 2.1.1 + # package to sync from a variety of cloud blob storage python-rclone == 0.0.2 @@ -30,6 +33,7 @@ numpy == 1.18.1 scikit-learn == 0.22.1 scipy == 1.4.1 pandas == 1.0.1 +spacy == 2.2.4 # packages for image processing Pillow == 7.0.0 @@ -37,6 +41,7 @@ Pillow == 7.0.0 # Etc pymongo == 3.10.1 redis == 3.4.1 +google-cloud-storage == 1.28.1 pika == 1.1.0 elasticsearch == 7.5.1 cassandra-driver == 3.21.0 diff --git a/core/python39Action/Dockerfile b/core/python39Action/Dockerfile index 713a2ea2..3eb96fcc 100644 --- a/core/python39Action/Dockerfile +++ b/core/python39Action/Dockerfile @@ -16,16 +16,17 @@ # # build go proxy from source -FROM golang:1.16 AS builder_source -ARG GO_PROXY_GITHUB_USER=apache -ARG GO_PROXY_GITHUB_BRANCH=master +ARG GO_PROXY_BASE_IMAGE=golang:1.18 +FROM $GO_PROXY_BASE_IMAGE AS builder_source +ARG GO_PROXY_GITHUB_USER=nimbella-corp +ARG GO_PROXY_GITHUB_BRANCH=dev RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} \ - https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src ;\ - cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \ - mv proxy /bin/proxy + https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src ;\ + cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \ + mv proxy /bin/proxy # or build it from a release -FROM golang:1.16 AS builder_release +FROM $GO_PROXY_BASE_IMAGE AS builder_release ARG GO_PROXY_RELEASE_VERSION=1.16@1.18.0 RUN curl -sL \ https://github.com/apache/openwhisk-runtime-go/archive/{$GO_PROXY_RELEASE_VERSION}.tar.gz\ @@ -42,6 +43,13 @@ ARG GO_PROXY_BUILD_FROM=release COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt +# install the functions-deployer +ARG DEPLOYER_DOWNLOAD +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -L ${DEPLOYER_DOWNLOAD} | tar xzf - \ + && rm -fr /usr/local/lib/dosls && mv dosls /usr/local/lib \ + && rm -f /usr/local/bin/dosls && ln -s /usr/local/lib/dosls/bootstrap /usr/local/bin/dosls + RUN mkdir -p /action WORKDIR / COPY --from=builder_source /bin/proxy /bin/proxy_source @@ -49,14 +57,17 @@ COPY --from=builder_release /bin/proxy /bin/proxy_release RUN mv /bin/proxy_${GO_PROXY_BUILD_FROM} /bin/proxy ADD bin/compile /bin/compile ADD lib/launcher.py /lib/launcher.py +ADD lib/prelauncher.py /lib/prelauncher.py # log initialization errors ENV OW_LOG_INIT_ERROR=1 # the launcher must wait for an ack ENV OW_WAIT_FOR_ACK=1 # using the runtime name to identify the execution environment -ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.9 +#ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.9 # compiler script ENV OW_COMPILER=/bin/compile +ENV OW_INIT_IN_ACTIONLOOP=/lib/prelauncher.py + ENTRYPOINT ["/bin/proxy"] diff --git a/core/python39Action/requirements.txt b/core/python39Action/requirements.txt index 549a80fc..d2e89fa2 100644 --- a/core/python39Action/requirements.txt +++ b/core/python39Action/requirements.txt @@ -10,3 +10,19 @@ simplejson == 3.17.2 virtualenv == 20.4.7 twisted == 21.2.0 netifaces == 0.11.0 + +# package to sync from a variety of cloud blob storage +python-rclone == 0.0.2 +google-cloud-storage == 1.35.1 + +# Nimbella package +nimbella == 2.1.1 + +# Etc +pymongo == 3.11.4 +redis == 3.4.1 +pika == 1.2.0 +elasticsearch == 7.13.2 +cassandra-driver == 3.25.0 +etcd3 == 0.12.0 +twilio == 6.60.0 diff --git a/core/python3Action/Dockerfile b/core/python3Action/Dockerfile index ef4615a3..a5ea67a3 100644 --- a/core/python3Action/Dockerfile +++ b/core/python3Action/Dockerfile @@ -17,8 +17,8 @@ # build go proxy from source FROM golang:1.16 AS builder_source -ARG GO_PROXY_GITHUB_USER=apache -ARG GO_PROXY_GITHUB_BRANCH=master +ARG GO_PROXY_GITHUB_USER=nimbella-corp +ARG GO_PROXY_GITHUB_BRANCH=dev RUN git clone --branch ${GO_PROXY_GITHUB_BRANCH} \ https://github.com/${GO_PROXY_GITHUB_USER}/openwhisk-runtime-go /src ;\ cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \ @@ -42,6 +42,11 @@ ARG GO_PROXY_BUILD_FROM=release COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt +# install nim +ARG NIM_INSTALL_SCRIPT=https://apigcp.nimbella.io/downloads/nim/nim-install-linux.sh +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl ${NIM_INSTALL_SCRIPT} | bash + RUN mkdir -p /action WORKDIR / COPY --from=builder_source /bin/proxy /bin/proxy_source @@ -54,8 +59,8 @@ ADD lib/launcher.py /lib/launcher.py ENV OW_LOG_INIT_ERROR=1 # the launcher must wait for an ack ENV OW_WAIT_FOR_ACK=1 -# execution environment -ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.7 +# using the runtime name to identify the execution environment +#ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.7 # compiler script ENV OW_COMPILER=/bin/compile diff --git a/core/python3Action/bin/compile b/core/python3Action/bin/compile index 052f2c4e..4b7180fd 100755 --- a/core/python3Action/bin/compile +++ b/core/python3Action/bin/compile @@ -117,6 +117,15 @@ def check(tgt_dir, module_name): if mod: with open(mod.origin, "rb") as f: ast.parse(f.read().decode("utf-8")) + # check virtualenv + path_to_virtualenv = abspath('%s/virtualenv' % tgt_dir) + if os.path.isdir(path_to_virtualenv): + activate_this_file = path_to_virtualenv + '/bin/activate_this.py' + if not os.path.exists(activate_this_file): + # check if this was packaged for windows + activate_this_file = path_to_virtualenv + '/Scripts/activate_this.py' + if not os.path.exists(activate_this_file): + sys.stderr.write("Invalid virtualenv. Zip file does not include 'activate_this.py'.\n") else: sys.stderr.write("Zip file does not include %s\n" % module_name) except SyntaxError as er: diff --git a/core/python3Action/lib/launcher.py b/core/python3Action/lib/launcher.py index ccf0c680..c7adc82f 100755 --- a/core/python3Action/lib/launcher.py +++ b/core/python3Action/lib/launcher.py @@ -19,7 +19,9 @@ from sys import stdout from sys import stderr from os import fdopen -import sys, os, json, traceback, warnings +import sys, os, json, traceback, time + +log_sentinel="XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX\n" try: # if the directory 'virtualenv' is extracted out of a zip file @@ -43,6 +45,35 @@ # now import the action as process input/output from main__ import main as main +class Context: + def __init__(self, env): + self.function_name = env["__OW_ACTION_NAME"] + self.function_version = env["__OW_ACTION_VERSION"] + self.activation_id = env["__OW_ACTIVATION_ID"] + self.request_id = env["__OW_TRANSACTION_ID"] + self.deadline = int(env["__OW_DEADLINE"]) + self.api_host = env["__OW_API_HOST"] + self.api_key = env.get("__OW_API_KEY", "") + self.namespace = env["__OW_NAMESPACE"] + + def get_remaining_time_in_millis(self): + epoch_now_in_ms = int(time.time() * 1000) + delta_ms = self.deadline - epoch_now_in_ms + return delta_ms if delta_ms > 0 else 0 + +def fun(payload, env): + # Compatibility: Supporting "old" context-less functions that have no params + # to match other languages. + if main.__code__.co_argcount == 0: + return main() or {} + + # Compatibility: Supports "old" context-less functions. + if main.__code__.co_argcount == 1: + return main(payload) or {} + + # Lambda-like "new-style" function. + return main(payload, Context(env)) or {} + out = fdopen(3, "wb") if os.getenv("__OW_WAIT_FOR_ACK", "") != "": out.write(json.dumps({"ok": True}, ensure_ascii=False).encode('utf-8')) @@ -62,12 +93,14 @@ env["__OW_%s" % key.upper()]= args[key] res = {} try: - res = main(payload) + res = fun(payload, env) except Exception as ex: print(traceback.format_exc(), file=stderr) res = {"error": str(ex)} out.write(json.dumps(res, ensure_ascii=False).encode('utf-8')) out.write(b'\n') + stdout.write(log_sentinel) + stderr.write(log_sentinel) stdout.flush() stderr.flush() out.flush() diff --git a/core/python3Action/lib/prelauncher.py b/core/python3Action/lib/prelauncher.py new file mode 100755 index 00000000..88189c91 --- /dev/null +++ b/core/python3Action/lib/prelauncher.py @@ -0,0 +1,150 @@ +#!/usr/local/bin/python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import print_function +from sys import stdin +from sys import stdout +from sys import stderr +from os import fdopen +import sys, os, json, traceback, base64, io, zipfile, time + +log_sentinel="XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX\n" +def write_sentinels(): + stdout.write(log_sentinel) + stderr.write(log_sentinel) + stdout.flush() + stderr.flush() + +out = fdopen(3, "wb") +def write_result(res): + out.write(json.dumps(res, ensure_ascii=False).encode('utf-8')) + out.write(b'\n') + out.flush() + +def cannot_start(msg): + stderr.write(msg) + write_result({"error": "Cannot start function. Check logs for details."}) + write_sentinels() + sys.exit(1) + +# Notify the Golang proxy that the process is awaiting input. +write_result({"ok": True}) + +# Read the init payload. +line = stdin.readline() +init = json.loads(line)["value"] + +# Set the environment from the init payload. +if init["env"]: + for key, value in init["env"].items(): + if isinstance(value, str): + os.environ[key] = value + else: + # Anything that's not a string needs to be stringified. + os.environ[key] = json.dumps(value) + +if init["binary"]: + # We have a base64 encoded zip as code. + buffer = base64.b64decode(init["code"]) + with zipfile.ZipFile(io.BytesIO(buffer)) as zip_ref: + zip_ref.extractall(".") + + # Note: We're ignoring `exec` here as we don't need a starter script. + if os.path.exists("__main__.py"): + os.rename("__main__.py", "main__.py") + if not os.path.exists("main__.py"): + cannot_start("Zip file does not include '__main__.py'.\n") + + try: + # If the directory 'virtualenv' is extracted out of a zip file. + path_to_virtualenv = os.path.abspath('virtualenv') + if os.path.isdir(path_to_virtualenv): + # Activate the virtualenv using activate_this.py contained in the virtualenv. + activate_this_file = path_to_virtualenv + '/bin/activate_this.py' + if not os.path.exists(activate_this_file): # try windows path + activate_this_file = path_to_virtualenv + '/Scripts/activate_this.py' + if os.path.exists(activate_this_file): + exec(open(activate_this_file).read(), {'__file__': activate_this_file}) + else: + cannot_start("Invalid virtualenv: Zip file does not include 'activate_this.py'.\n") + except Exception as ex: + traceback.print_exc(file=stderr, limit=0) + cannot_start("Invalid virtualenv: Failed to active virtualenv %s.\n" % str(ex)) +else: + # TODO: We can optimize this further by compiling the code in-process. + with open("main__.py", mode="wb") as f: + f.write(init["code"].encode("utf-8")) + +# Import the action itself. +try: + sys.path.append(os.getcwd()) + main = getattr(__import__("main__", fromlist=[init["main"]]), init["main"]) +except Exception as ex: + cannot_start("Invalid function: %s\n" % str(ex)) + +class Context: + def __init__(self, env): + self.function_name = env["__OW_ACTION_NAME"] + self.function_version = env["__OW_ACTION_VERSION"] + self.activation_id = env["__OW_ACTIVATION_ID"] + self.request_id = env["__OW_TRANSACTION_ID"] + self.deadline = int(env["__OW_DEADLINE"]) + self.api_host = env["__OW_API_HOST"] + self.api_key = env.get("__OW_API_KEY", "") + self.namespace = env["__OW_NAMESPACE"] + + def get_remaining_time_in_millis(self): + epoch_now_in_ms = int(time.time() * 1000) + delta_ms = self.deadline - epoch_now_in_ms + return delta_ms if delta_ms > 0 else 0 + +def fun(payload, env): + # Compatibility: Supporting "old" context-less functions that have no params + # to match other languages. + if main.__code__.co_argcount == 0: + return main() or {} + + # Compatibility: Supports "old" context-less functions. + if main.__code__.co_argcount == 1: + return main(payload) or {} + + # Lambda-like "new-style" function. + return main(payload, Context(env)) or {} + +# Acknowledge the initialization. +write_result({"ok": True}) + +# Enter the actual action loop. +while True: + line = stdin.readline() + if not line: break + args = json.loads(line) + payload = {} + for key in args: + if key == "value": + payload = args["value"] + else: + os.environ["__OW_%s" % key.upper()]= args[key] + res = {} + try: + res = fun(payload, os.environ) + except Exception as ex: + print(traceback.format_exc(), file=stderr) + res = {"error": str(ex)} + write_result(res) + write_sentinels() diff --git a/core/python3Action/requirements.txt b/core/python3Action/requirements.txt index d60e88a8..027f954e 100644 --- a/core/python3Action/requirements.txt +++ b/core/python3Action/requirements.txt @@ -1,11 +1,28 @@ # default available packages for python3action -beautifulsoup4 == 4.6.3 -httplib2 == 0.11.3 -kafka_python == 1.4.3 -lxml == 4.2.5 -python-dateutil == 2.7.3 -requests == 2.19.1 -scrapy == 1.5.1 -simplejson == 3.16.0 -virtualenv == 16.0.0 -twisted == 18.7.0 +beautifulsoup4 == 4.8.2 +httplib2 == 0.17.0 +kafka_python == 1.4.7 +lxml == 4.5.0 +python-dateutil == 2.8.1 +requests == 2.22.0 +scrapy == 1.8.0 +simplejson == 3.17.0 +virtualenv == 16.7.9 +twisted == 19.10.0 +netifaces == 0.10.9 + +# package to sync from a variety of cloud blob storage +python-rclone == 0.0.2 +google-cloud-storage == 1.28.1 + +# Nimbella package +nimbella == 2.1.1 + +# Etc +pymongo == 3.10.1 +redis == 3.4.1 +pika == 1.1.0 +elasticsearch == 7.5.1 +cassandra-driver == 3.21.0 +etcd3 == 0.11.1 +twilio == 6.35.4 diff --git a/settings.gradle b/settings.gradle index 82b9a9ae..5e6a8118 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,9 +17,11 @@ include 'tests' +include 'core:python2ActionLoop' include 'core:python3Action' include 'core:python36AiAction' include 'core:python39Action' +include 'core:python311Action' rootProject.name = 'runtime-python' @@ -33,13 +35,10 @@ gradle.ext.scala = [ compileFlags: ['-feature', '-unchecked', '-deprecation', '-Xfatal-warnings', '-Ywarn-unused-import'] ] -gradle.ext.akka = [version : '2.6.12'] -gradle.ext.akka_http = [version : '10.2.4'] - gradle.ext.scalafmt = [ - version: '1.5.0', + version: '1.16.2', config: new File(rootProject.projectDir, '.scalafmt.conf') ] -gradle.ext.akka = [version: '2.6.12'] -gradle.ext.akka_http = [version: '10.2.4'] +gradle.ext.akka = [version: '2.5.32'] +gradle.ext.akka_http = [version: '10.1.15'] diff --git a/tests/src/test/resources/build.sh b/tests/src/test/resources/build.sh index bbc46160..0c9391a7 100755 --- a/tests/src/test/resources/build.sh +++ b/tests/src/test/resources/build.sh @@ -21,7 +21,7 @@ if [ -f ".built" ]; then exit 0 fi -for i in v3.7 v3.6-ai v3.9 +for i in v3.7 v3.6-ai v3.9 v3.11 do echo "*** $i ***" zip -r -j - python_virtualenv | docker run -i action-python-$i -compile main >python-${i}_virtualenv.zip cp python-${i}_virtualenv.zip python-${i}_virtualenv_invalid_main.zip diff --git a/tests/src/test/scala/runtime/actionContainers/Python311Tests.scala b/tests/src/test/scala/runtime/actionContainers/Python311Tests.scala new file mode 100644 index 00000000..e758872e --- /dev/null +++ b/tests/src/test/scala/runtime/actionContainers/Python311Tests.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime.actionContainers + +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class Python311Tests extends Python37Tests { + + override lazy val imageName = "action-python-v3.11" + + override lazy val zipPrefix = "python-v3.11" + + override lazy val errorCodeOnRun = false + + override val testNoSource = TestConfig("", hasCodeStub = false) +} diff --git a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala index d4bd9d9a..1417c977 100644 --- a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala +++ b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala @@ -16,7 +16,9 @@ */ package runtime.actionContainers -import spray.json.{JsObject, JsString} +import spray.json._ +import spray.json.DefaultJsonProtocol._ +import java.time.Instant trait PythonAdvancedTests { this: PythonBasicTests => @@ -58,7 +60,7 @@ trait PythonAdvancedTests { // action loop detects those errors at init time val (initCode, initRes) = c.init(initPayload(code)) initCode should be(502) - initRes.get.fields.get("error").get.toString() should include("Cannot start action") + initRes.get.fields.get("error").get.toString() should include("Cannot start function") } checkStreams(out, err, { case (o, e) => @@ -91,4 +93,58 @@ trait PythonAdvancedTests { e shouldBe empty }) } + + Map( + "prelaunched" -> Map.empty[String, String], + "non-prelaunched" -> Map("OW_INIT_IN_ACTIONLOOP" -> ""), + ).foreach { case (name, env) => + it should s"support a function with a lambda-like signature $name" in { + val (out, err) = withActionContainer(env + ("__OW_API_HOST" -> "testhost")) { c => + val code = + """ + |def main(event, context): + | return { + | "remaining_time": context.get_remaining_time_in_millis(), + | "activation_id": context.activation_id, + | "request_id": context.request_id, + | "function_name": context.function_name, + | "function_version": context.function_version, + | "api_host": context.api_host, + | "api_key": context.api_key, + | "namespace": context.namespace + | } + """.stripMargin + + val (initCode, _) = c.init(initPayload(code)) + initCode should be(200) + + val (runCode, out) = c.run(runPayload( + JsObject(), + Some(JsObject( + "deadline" -> Instant.now.plusSeconds(10).toEpochMilli.toString.toJson, + "activation_id" -> "testaid".toJson, + "transaction_id" -> "testtid".toJson, + "action_name" -> "testfunction".toJson, + "action_version" -> "0.0.1".toJson, + "namespace" -> "testnamespace".toJson, + "api_key" -> "testkey".toJson + )) + )) + runCode should be(200) + + val remainingTime = out.get.fields("remaining_time").convertTo[Int] + remainingTime should be > 9500 // We give the test 500ms of slack to invoke the function to avoid flakes. + out shouldBe Some(JsObject( + "remaining_time" -> remainingTime.toJson, + "activation_id" -> "testaid".toJson, + "request_id" -> "testtid".toJson, + "function_name" -> "testfunction".toJson, + "function_version" -> "0.0.1".toJson, + "api_host" -> "testhost".toJson, + "api_key" -> "testkey".toJson, + "namespace" -> "testnamespace".toJson + )) + } + } + } }