diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3d29a9a..9076a192 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,26 +423,15 @@ jobs: run: PATH="$PATH:/c/Users/runneradmin/.cargo/bin" nox -s test-examples test-emscripten: - name: Test Emscripten runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v7 - uses: actions/setup-node@v6 with: - node-version: 18 + node-version: 22 - run: | - PYODIDE_VERSION=0.21.0 - - cd emscripten - npm i pyodide@0.21.0 - cd node_modules/pyodide/ - npx -y prettier -w pyodide.asm.js - EMSCRIPTEN_VERSION=$(node -p "require('./repodata.json').info.platform.split('_').slice(1).join('.')") - PYTHON_VERSION=3.10 - - echo "PYODIDE_VERSION=$PYODIDE_VERSION" >> $GITHUB_ENV - echo "EMSCRIPTEN_VERSION=$EMSCRIPTEN_VERSION" >> $GITHUB_ENV - echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV + uvx nox -s install-pyodide-emscripten echo "ORIG_PATH=$PATH" >> $GITHUB_ENV - uses: dtolnay/rust-toolchain@nightly with: @@ -452,11 +441,6 @@ jobs: with: version: ${{env.EMSCRIPTEN_VERSION}} actions-cache-folder: emsdk-cache - - uses: actions/setup-python@v6 - id: setup-python - with: - python-version: ${{env.PYTHON_VERSION}} - - run: pip install nox - uses: actions/cache@v5 with: path: | @@ -466,4 +450,6 @@ jobs: - name: Test run: | export PATH=$ORIG_PATH:$PATH - nox -s test-examples-emscripten + uvx nox -s test-examples-emscripten + env: + UV_PYTHON: ${{ env.PYTHON_VERSION }} diff --git a/emscripten/_sysconfigdata__emscripten_wasm32-emscripten.py b/emscripten/_sysconfigdata__emscripten_wasm32-emscripten.py index 6596c0ac..3c3b21d9 100644 --- a/emscripten/_sysconfigdata__emscripten_wasm32-emscripten.py +++ b/emscripten/_sysconfigdata__emscripten_wasm32-emscripten.py @@ -8,9 +8,9 @@ "CCSHARED": "", "CFLAGS": "-Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g " "-fwrapv -O3 -Wall -O2 -g0 -fPIC", - "EXT_SUFFIX": ".cpython-310-wasm32-emscripten.so", + "EXT_SUFFIX": ".cpython-313-wasm32-emscripten.so", "HOST_GNU_TYPE": "wasm32-unknown-emscripten", "LDSHARED": "emcc -sSIDE_MODULE=1", "Py_DEBUG": "0", - "py_version_nodot": "310", + "py_version_nodot": "313", } diff --git a/emscripten/get_emscripten_version.js b/emscripten/get_emscripten_version.js new file mode 100644 index 00000000..ae85ccc4 --- /dev/null +++ b/emscripten/get_emscripten_version.js @@ -0,0 +1,8 @@ +import { loadPyodide } from "pyodide"; + +const pyodide = await loadPyodide(); + +pyodide.runPython(` +import platform +print(platform.platform().split("-")[1]) +`); diff --git a/emscripten/get_python_version.js b/emscripten/get_python_version.js new file mode 100644 index 00000000..8ea8f94d --- /dev/null +++ b/emscripten/get_python_version.js @@ -0,0 +1,9 @@ +import { loadPyodide } from "pyodide"; + +const pyodide = await loadPyodide(); + +pyodide.runPython(` +import sys +major, minor = sys.version_info[:2] +print(f"{major}.{minor}") +`); diff --git a/emscripten/package-lock.json b/emscripten/package-lock.json new file mode 100644 index 00000000..7f375544 --- /dev/null +++ b/emscripten/package-lock.json @@ -0,0 +1,50 @@ +{ + "name": "emscripten", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "pyodide": "^0.29.1" + } + }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, + "node_modules/pyodide": { + "version": "0.29.1", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-0.29.1.tgz", + "integrity": "sha512-mpk9jtkiM7Ugh1r9P9dbR8vKrmf0lED32hBZq+Fn1kkkBiUoOjSsJEWcyprugICpiFpIXpUOf80ZrvFXkQMk2g==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/ws": { + "version": "8.8.1", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/emscripten/package.json b/emscripten/package.json new file mode 100644 index 00000000..480d76dc --- /dev/null +++ b/emscripten/package.json @@ -0,0 +1,6 @@ +{ + "type": "module", + "dependencies": { + "pyodide": "^0.29.1" + } +} diff --git a/emscripten/pyo3_config.ini b/emscripten/pyo3_config.ini deleted file mode 100644 index a91b4f9c..00000000 --- a/emscripten/pyo3_config.ini +++ /dev/null @@ -1,7 +0,0 @@ -implementation=CPython -version=3.10 -shared=true -abi3=false -lib_name=python3.10 -pointer_width=32 -suppress_build_script_link_lines=false diff --git a/emscripten/runner.js b/emscripten/runner.js index 68bf0ea1..bba3a086 100644 --- a/emscripten/runner.js +++ b/emscripten/runner.js @@ -1,5 +1,5 @@ -const { opendir } = require("node:fs/promises"); -const { loadPyodide } = require("pyodide"); +import { opendir } from "node:fs/promises"; +import { loadPyodide } from "pyodide"; async function findWheel(distDir) { const dir = await opendir(distDir); @@ -10,7 +10,7 @@ async function findWheel(distDir) { } } -function make_tty_ops(stream){ +function make_tty_ops(stream) { return { // get_char has 3 particular return values: // a.) the next character represented as an integer @@ -32,14 +32,14 @@ function make_tty_ops(stream){ }, put_char(tty, val) { try { - if(val !== null){ + if (val !== null) { tty.output.push(val); } if (val === null || val === 10) { process.stdout.write(Buffer.from(tty.output)); tty.output = []; } - } catch(e){ + } catch (e) { console.warn(e); } }, @@ -49,32 +49,32 @@ function make_tty_ops(stream){ } stream.write(Buffer.from(tty.output)); tty.output = []; - } + }, + fsync(tty) {}, }; } -function setupStreams(FS, TTY){ +function setupStreams(FS, TTY) { let mytty = FS.makedev(FS.createDevice.major++, 0); let myttyerr = FS.makedev(FS.createDevice.major++, 0); - TTY.register(mytty, make_tty_ops(process.stdout)) - TTY.register(myttyerr, make_tty_ops(process.stderr)) - FS.mkdev('/dev/mytty', mytty); - FS.mkdev('/dev/myttyerr', myttyerr); - FS.unlink('/dev/stdin'); - FS.unlink('/dev/stdout'); - FS.unlink('/dev/stderr'); - FS.symlink('/dev/mytty', '/dev/stdin'); - FS.symlink('/dev/mytty', '/dev/stdout'); - FS.symlink('/dev/myttyerr', '/dev/stderr'); + TTY.register(mytty, make_tty_ops(process.stdout)); + TTY.register(myttyerr, make_tty_ops(process.stderr)); + FS.mkdev("/dev/mytty", mytty); + FS.mkdev("/dev/myttyerr", myttyerr); + FS.unlink("/dev/stdin"); + FS.unlink("/dev/stdout"); + FS.unlink("/dev/stderr"); + FS.symlink("/dev/mytty", "/dev/stdin"); + FS.symlink("/dev/mytty", "/dev/stdout"); + FS.symlink("/dev/myttyerr", "/dev/stderr"); FS.closeStream(0); FS.closeStream(1); FS.closeStream(2); - var stdin = FS.open('/dev/stdin', 0); - var stdout = FS.open('/dev/stdout', 1); - var stderr = FS.open('/dev/stderr', 1); + var stdin = FS.open("/dev/stdin", 0); + var stdout = FS.open("/dev/stdout", 1); + var stderr = FS.open("/dev/stderr", 1); } - const pkgDir = process.argv[2]; const distDir = pkgDir + "/dist"; const testDir = pkgDir + "/tests"; @@ -85,9 +85,9 @@ async function main() { let errcode = 1; try { - pyodide = await loadPyodide(); + const pyodide = await loadPyodide(); const FS = pyodide.FS; - setupStreams(FS, pyodide._module.TTY); + // setupStreams(FS, pyodide._module.TTY); const NODEFS = FS.filesystems.NODEFS; FS.mkdir("/test_dir"); FS.mount(NODEFS, { root: testDir }, "/test_dir"); diff --git a/noxfile.py b/noxfile.py index e86a0c1a..cd874470 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,9 +1,11 @@ import os +from contextlib import ExitStack from inspect import cleandoc as heredoc from glob import glob from pathlib import Path import shutil import sys +import tempfile import nox import nox.command @@ -227,10 +229,40 @@ def test(session: nox.Session): session.run("pytest", "setuptools_rust", "tests", *session.posargs) +PYODIDE_VERSION = "0.29.1" +EMSCRIPTEN_DIR = Path("./emscripten").resolve() + + +@nox.session(name="install-pyodide-emscripten") +def install_pyodide_emscripten(session: nox.Session): + with session.chdir(EMSCRIPTEN_DIR): + session.run("npm", "install", f"pyodide@{PYODIDE_VERSION}", external=True) + emscripten_version = session.run( + "node", "get_emscripten_version.js", external=True, silent=True + ).strip() + python_version = session.run( + "node", "get_python_version.js", external=True, silent=True + ).strip() + + with ExitStack() as stack: + if "GITHUB_ENV" in os.environ: + out = stack.enter_context(open(os.environ["GITHUB_ENV"], "a")) + else: + out = sys.stdout + + print(f"PYODIDE_VERSION={PYODIDE_VERSION}", file=out) + print(f"EMSCRIPTEN_VERSION={emscripten_version}", file=out) + print(f"PYTHON_VERSION={python_version}", file=out) + + if "GITHUB_ENV" not in os.environ: + print( + "You will need to install emscripten yourself to match the target version." + ) + + @nox.session(name="test-examples-emscripten") def test_examples_emscripten(session: nox.Session): session.install(".", "build") - emscripten_dir = Path("./emscripten").resolve() session.run( "rustup", @@ -246,25 +278,41 @@ def test_examples_emscripten(session: nox.Session): examples_dir / "html-py-ever", examples_dir / "namespace_package", ] - for example in test_crates: - env = os.environ.copy() - env.update( - RUSTUP_TOOLCHAIN="nightly", - PYTHONPATH=str(emscripten_dir), - _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata__emscripten_wasm32-emscripten", - _PYTHON_HOST_PLATFORM="emscripten_3_1_14_wasm32", - CARGO_BUILD_TARGET="wasm32-unknown-emscripten", - CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=str( - emscripten_dir / "emcc_wrapper.py" - ), - PYO3_CONFIG_FILE=str(emscripten_dir / "pyo3_config.ini"), - ) - with session.chdir(example): - cmd = ["python", "-m", "build", "--wheel", "--no-isolation"] - session.run(*cmd, env=env, external=True) - - with session.chdir(emscripten_dir): - session.run("node", "runner.js", str(example), external=True) + + python_version = os.environ["PYTHON_VERSION"] + emscripten_version = os.environ["EMSCRIPTEN_VERSION"] + + with tempfile.NamedTemporaryFile("w") as pyo3_config: + pyo3_config.write(f"""\ +implementation=CPython +version={python_version} +shared=true +abi3=false +pointer_width=32 +""") + pyo3_config.flush() + + emscripten_version_joined = emscripten_version.replace(".", "_") + + for example in test_crates: + env = os.environ.copy() + env.update( + RUSTUP_TOOLCHAIN="nightly", + PYTHONPATH=str(EMSCRIPTEN_DIR), + _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata__emscripten_wasm32-emscripten", + _PYTHON_HOST_PLATFORM=f"emscripten_{emscripten_version_joined}_wasm32", + CARGO_BUILD_TARGET="wasm32-unknown-emscripten", + CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=str( + EMSCRIPTEN_DIR / "emcc_wrapper.py" + ), + PYO3_CONFIG_FILE=pyo3_config.name, + ) + with session.chdir(example): + cmd = ["python", "-m", "build", "--wheel", "--no-isolation"] + session.run(*cmd, env=env, external=True) + + with session.chdir(EMSCRIPTEN_DIR): + session.run("node", "runner.js", str(example), external=True) @nox.session(name="bump-version")