diff --git a/.devcontainer.json b/.devcontainer.json index fe810f4..f5facbf 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -22,7 +22,8 @@ "vscode": { "extensions": [ "ms-python.python", - "ms-python.black-formatter" + "ms-python.black-formatter", + "ms-vscode.cpptools-extension-pack" ], "settings": { "python.analysis.typeCheckingMode": "strict", diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9aca3e1..508143e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,51 +17,91 @@ jobs: with: release-type: python - - uses: actions/checkout@v3 - if: ${{ steps.release.outputs.release_created }} + build_wheels: + name: Build wheels for ${{ matrix.arch }} ${{ matrix.os }} py3.${{ matrix.python }} + needs: release + runs-on: ${{ matrix.os }}-latest + if: ${{ needs.release.outputs.release_created }} + strategy: + fail-fast: false + matrix: + python: [9, 10, 11, 12, 13] + os: [ubuntu, macos] + arch: [x86_64, aarch64, universal2] + exclude: + - os: macos + arch: universal2 + python: 7 + - os: ubuntu + arch: universal2 + - os: macos + arch: x86_64 + - os: macos + arch: aarch64 + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU for aarch64 emulation + if: runner.os == 'Linux' && matrix.arch == 'aarch64' + uses: docker/setup-qemu-action@v3 with: - fetch-depth: 2 + platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.19.1 + env: + CIBW_SKIP: "*-musllinux_* pp3*-manylinux_aarch64" + CIBW_BUILD: ${{ format('*p3{0}-*', matrix.python) }} + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {package}/tests" - - name: Set up Python - if: ${{ steps.release.outputs.release_created }} - uses: actions/setup-python@v4 + - uses: actions/upload-artifact@v4 with: - python-version: '3.9' # keep synced with dev-env.yml + name: wheels-${{ matrix.os }}-${{ matrix.arch }}-3${{ matrix.python }} + path: ./wheelhouse/*.whl - - name: Upgrade pip - if: ${{ steps.release.outputs.release_created }} - run: | - pip install --upgrade pip - pip --version + merge_wheels: + runs-on: ubuntu-latest + needs: build_wheels + steps: + - name: Merge wheel artifacts into a single artifact + uses: actions/upload-artifact/merge@v4 + with: + name: wheels + pattern: wheels-* + delete-merged: true - - name: Install Poetry - if: ${{ steps.release.outputs.release_created }} - run: | - pip install 'poetry==1.8.3' # keep version synced with dev-env.yml - poetry --version + build_sdist: + needs: release + runs-on: ubuntu-latest + if: ${{ needs.release.outputs.release_created }} + env: + job_python_version: "3.9" # keep synced with dev-env.yml - - name: Bump version for developmental release - if: ${{ steps.release.outputs.release_created }} - env: - version: ${{ steps.release.outputs.tag_name }} - run: | - poetry version $version + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ env.job_python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.job_python_version }} - - name: Build package - if: ${{ steps.release.outputs.release_created }} + - name: Create source distribution run: | - poetry build --ansi + pip install build + python -m build --sdist . - uses: actions/upload-artifact@v4 - if: ${{ steps.release.outputs.release_created }} with: - name: dist - path: dist/ + name: sdist + path: dist/panct-*.tar.gz upload_pypi: - needs: release + needs: [build_wheels, build_sdist] runs-on: ubuntu-latest - if: ${{ needs.release.outputs.release_created }} environment: release permissions: # IMPORTANT: this permission is mandatory for trusted publishing diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 40b4e83..50d12f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,7 @@ jobs: auto-activate-base: false miniforge-version: latest use-mamba: true + conda-remove-defaults: "true" - name: Get Date id: get-date @@ -48,15 +49,19 @@ jobs: with: path: ${{ env.CONDA }}/envs key: - conda-${{ runner.os }}--${{ runner.arch }}--${{ steps.get-date.outputs.today }}-${{ hashFiles('dev-env.yml') }}-${{ env.CACHE_NUMBER }} + conda-${{ runner.os }}--${{ runner.arch }}--${{ steps.get-date.outputs.today }}-${{ hashFiles('dev-env.yml') }}-${{ matrix.python }}-${{ env.CACHE_NUMBER }} env: # Increase this value to reset cache if dev-env.yml has not changed CACHE_NUMBER: 0 id: cache - name: Install dev environment - run: - mamba env update -n panct -f dev-env.yml + env: + PYTHON_VERSION: ${{ matrix.python }} + run: | + # sync the python version in the dev-env.yml file + sed s'/python=3.[[:digit:]]\+ /python='"$PYTHON_VERSION"' /' dev-env.yml > dev-env.new.yml + mamba env update -n panct -f dev-env.new.yml if: steps.cache.outputs.cache-hit != 'true' - name: Try to build panct diff --git a/.gitignore b/.gitignore index c2dae4e..caa3eea 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,38 @@ fil-result/ # OSX *.DS_Store* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so +*.pyd + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Cython +panct/*.c +panct/*.cpp +panct/*.html diff --git a/NOTES.md b/NOTES.md new file mode 100644 index 0000000..a875fa6 --- /dev/null +++ b/NOTES.md @@ -0,0 +1,2 @@ +1. How do we want to include the gbz-base source code in the repo? Options: git submodule or install as setup step +2. Do we want to automatically skip compilation of the cython module if it doesn't work? Or should we just fail/error explicitly. The former could be helpful if we want to implement some kind of slower alternative diff --git a/build.py b/build.py new file mode 100644 index 0000000..32f8150 --- /dev/null +++ b/build.py @@ -0,0 +1,85 @@ +# type: ignore +import os +import shutil +from pathlib import Path + +# This file was adapted from https://github.com/BrianPugh/python-template/blob/main/build.py + +# Uncomment if library can still function if extensions fail to compile (e.g. slower, python fallback). +# Don't allow failure if cibuildwheel is running. +# allowed_to_fail = os.environ.get("CIBUILDWHEEL", "0") != "1" +allowed_to_fail = False + + +def build_cython_extensions(): + # when using setuptools, you should import setuptools before Cython, + # otherwise, both might disagree about the class to use. + from setuptools import Extension # noqa: I001 + from setuptools.dist import Distribution # noqa: I001 + import Cython.Compiler.Options # pyright: ignore [reportMissingImports] + from Cython.Build import ( + build_ext, + cythonize, + ) # pyright: ignore [reportMissingImports] + + Cython.Compiler.Options.annotate = True + + if os.name == "nt": # Windows + extra_compile_args = [ + "/O2", + ] + else: # UNIX-based systems + extra_compile_args = [ + "-O3", + "-Werror", + "-Wno-unreachable-code-fallthrough", + "-Wno-deprecated-declarations", + "-Wno-parentheses-equality", + ] + extra_compile_args.append("-UNDEBUG") # Cython disables asserts by default. + # Relative to project root director + include_dirs = [ + "panct/", + "panct/_c_src", + ] + + c_files = [str(x) for x in Path("panct/_c_src").rglob("*.c")] + extensions = [ + Extension( + # Your .pyx file will be available to cpython at this location. + "panct._c_extension", + [ + # ".c" and ".pyx" source file paths + "panct/_c_extension.pyx", + *c_files, + ], + include_dirs=include_dirs, + extra_compile_args=extra_compile_args, + language="c", + ), + ] + + include_dirs = set() + for extension in extensions: + include_dirs.update(extension.include_dirs) + include_dirs = list(include_dirs) + + ext_modules = cythonize( + extensions, include_path=include_dirs, language_level=3, annotate=True + ) + dist = Distribution({"ext_modules": ext_modules}) + cmd = build_ext(dist) + cmd.ensure_finalized() + cmd.run() + + for output in cmd.get_outputs(): + output = Path(output) + relative_extension = output.relative_to(cmd.build_lib) + shutil.copyfile(output, relative_extension) + + +try: + build_cython_extensions() +except Exception: + if not allowed_to_fail: + raise diff --git a/dev-env.yml b/dev-env.yml index 1851a10..b549d35 100644 --- a/dev-env.yml +++ b/dev-env.yml @@ -5,7 +5,7 @@ channels: dependencies: - conda-forge::python=3.9 # the lowest version of python that we formally support; keep in sync with release.yml, .readthedocs.yaml, and noxfile.py - conda-forge::pip==24.0 - - conda-forge::poetry==1.8.3 # should keep this in sync with version in release.yml + - conda-forge::poetry==1.8.3 - conda-forge::nox==2024.4.15 - conda-forge::poetry-plugin-export==1.8.0 - pip: diff --git a/noxfile.py b/noxfile.py index 0158b14..913c5cc 100644 --- a/noxfile.py +++ b/noxfile.py @@ -32,6 +32,14 @@ def docs(session: Session) -> None: if build_dir.exists(): shutil.rmtree(build_dir) + session.install(".") + session.install( + "sphinx", + "sphinx-autodoc-typehints", + "sphinx-rtd-theme", + "numpydoc", + "sphinx-click", + ) session.run("sphinx-build", *args) @@ -63,10 +71,12 @@ def install_handle_python_numpy(session): @session(venv_backend=conda_cmd, venv_params=conda_args, python=python_versions) def tests(session: Session) -> None: """Run the test suite.""" + # first, delete any existing envs to avoid + # https://github.com/cjolowicz/nox-poetry/issues/1188 + session.run("poetry", "env", "remove", "--all") session.conda_install( "coverage[toml]", "pytest", - "numpy>=1.20.0", channel="conda-forge", ) install_handle_python_numpy(session) @@ -83,6 +93,9 @@ def tests(session: Session) -> None: @session(python=python_versions) def tests(session: Session) -> None: """Run the test suite.""" + # first, delete any existing envs to avoid + # https://github.com/cjolowicz/nox-poetry/issues/1188 + session.run("poetry", "env", "remove", "--all") session.install("coverage[toml]", "pytest") install_handle_python_numpy(session) try: diff --git a/panct/__init__.py b/panct/__init__.py index 5d14e36..3180f9d 100644 --- a/panct/__init__.py +++ b/panct/__init__.py @@ -1,8 +1,4 @@ -try: - from importlib.metadata import version, PackageNotFoundError -except ImportError: - # handles py3.7, since importlib.metadata was introduced in py3.8 - from importlib_metadata import version, PackageNotFoundError +from importlib.metadata import version, PackageNotFoundError try: __version__ = version(__name__) diff --git a/panct/_c_extension.pyi b/panct/_c_extension.pyi new file mode 100644 index 0000000..7c01af1 --- /dev/null +++ b/panct/_c_extension.pyi @@ -0,0 +1,5 @@ +class Foo: + def __init__(self): ... + def __call__(self): ... + +def divide(x: float, y: float) -> float: ... diff --git a/panct/_c_extension.pyx b/panct/_c_extension.pyx new file mode 100644 index 0000000..f8000bc --- /dev/null +++ b/panct/_c_extension.pyx @@ -0,0 +1,40 @@ +cimport cpythontemplate # See cpythontemplate.pxd +# Invoke PyErr_CheckSignals() occasionally if your C code runs long. +# This allows your code to be interrupted via ctrl+c. +from cpython.exc cimport PyErr_CheckSignals +from cpython.mem cimport PyMem_Malloc, PyMem_Free +from libc.stddef cimport size_t + + +cdef class Foo: + """Pythonic interface to the C "foo" struct.""" + cdef cpythontemplate.foo_t * _object + + def __cinit__(self): + # Automatically called before __init__. + # All arguments passed to __init__ are also passed to __cinit__. + # * As a convenience, if __cinit__() takes no arguments (other than self), it will + # ignore arguments passed to the constructor without complaining about signature mismatch. + # Allocate memory for C objects here. + self._object = PyMem_Malloc(sizeof(cpythontemplate.foo_t)) + if self._object is NULL: + raise MemoryError + + def __dealloc__(self): + # Should "undo" __cinit__ + PyMem_Free(self._object) + + def __init__(self): + cpythontemplate.foo_init(self._object) + + def __call__(self): + # invoke increment + cpythontemplate.foo_increment(self._object) + +# Functions declared with cpdef are visible to both cython and python. +# https://cython.readthedocs.io/en/latest/src/userguide/language_basics.html#python-functions-vs-c-functions +# https://cython.readthedocs.io/en/latest/src/userguide/language_basics.html#error-return-values +cpdef float divide(float x, float y) except? 1.23: + if y == 0.0: + raise ZeroDivisionError + return x / y diff --git a/panct/_c_src/foo.c b/panct/_c_src/foo.c new file mode 100644 index 0000000..68ffdda --- /dev/null +++ b/panct/_c_src/foo.c @@ -0,0 +1,11 @@ +#include "foo.h" + +int foo_init(foo_t *foo){ + foo->counter = 0; + return FOO_OK; +} + +int foo_increment(foo_t *foo){ + foo->counter++; + return foo->counter; +} diff --git a/panct/_c_src/foo.h b/panct/_c_src/foo.h new file mode 100644 index 0000000..1562b85 --- /dev/null +++ b/panct/_c_src/foo.h @@ -0,0 +1,24 @@ +#ifndef FOO_H +#define FOO_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + FOO_OK = 0, +} foo_res_t; + +typedef struct { + int counter; +} foo_t; + +int foo_init(foo_t *foo); + +int foo_increment(foo_t *foo); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/panct/cpythontemplate.pxd b/panct/cpythontemplate.pxd new file mode 100644 index 0000000..129e145 --- /dev/null +++ b/panct/cpythontemplate.pxd @@ -0,0 +1,26 @@ +"""Primary cimport. + +This file acts as an interface between cython and C header files. + +Other pyx files will be able to "cimport cpythontemplate" to gain +access to functions and structures defined here. + +This file should be a simple translation from existing c header file(s). +Multiple header files may be translated here. +""" + +from libcpp cimport bool +from libc.stdint cimport uint8_t, uint32_t + +cdef extern from "foo.h": + # Translate typedef'd structs: + ctypedef struct foo_t: + int counter + + # Translate enums: + ctypedef enum foo_res_t: + FOO_OK = 0, + + # Typical function declaration + void foo_init(foo_t * foo); + void foo_increment(foo_t * foo); diff --git a/pyproject.toml b/pyproject.toml index d1dee9b..03af603 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=1.0.0", "Cython>=0.3.11", "setuptools>=75.1.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] @@ -12,6 +12,7 @@ repository = "https://github.com/CAST-genomics/panCT" homepage = "https://github.com/CAST-genomics/panCT" documentation = "https://panCT.readthedocs.io" readme = "README.md" +include = ["panct/*.so", "panct/*.pyd"] # Compiled extensions [tool.poetry.dependencies] python = ">=3.9" @@ -34,6 +35,10 @@ ipython = ">=7.34.0" coverage = {extras = ["toml"], version = ">=7.2.7"} filprofiler = ">=2023.3.1" +[tool.poetry.build] +generate-setup-file = false +script = "build.py" + [tool.poetry.scripts] panct = 'panct.__main__:app'