From 544ffdc9171c05071abcaf5f99af79e596e2980b Mon Sep 17 00:00:00 2001 From: Kostis Papazafeiropoulos Date: Fri, 31 Oct 2025 17:37:51 +0000 Subject: [PATCH 1/6] refactor(c_types): Enable setting the numeric C type from `CAny` Enable setting the C type for numeric type wrappers from the `CAny` adapter class. This allows functions that use `CAny`, instead of the type-specific `C*` classes, to specify the C types that will be used to represent numeric Python types. PR: https://github.com/nubificus/vaccel-python/pull/29 Signed-off-by: Kostis Papazafeiropoulos Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- vaccel/_c_types/types.py | 15 +++++++++++---- vaccel/_c_types/wrappers/cbytes.py | 8 ++++++-- vaccel/_c_types/wrappers/cfloat.py | 13 +++++++++---- vaccel/_c_types/wrappers/cint.py | 25 ++++++++++++++++++++++--- vaccel/_c_types/wrappers/clist.py | 3 ++- vaccel/_c_types/wrappers/cnumpyarray.py | 3 ++- vaccel/_c_types/wrappers/cstr.py | 8 ++++++-- 7 files changed, 58 insertions(+), 17 deletions(-) diff --git a/vaccel/_c_types/types.py b/vaccel/_c_types/types.py index 31163b1..978d4bf 100644 --- a/vaccel/_c_types/types.py +++ b/vaccel/_c_types/types.py @@ -75,13 +75,18 @@ class CAny(CType): _wrapped (CType): The wrapped C object. """ - def __init__(self, obj: Any): + def __init__(self, obj: Any, *, precision: str | None = None): """Initializes a new `CAny` object. Args: obj: The python object to be wrapped as a C type. + precision: The C type that will be used to represent the python + object type, if the type is numeric. """ - self._wrapped = to_ctype(obj) + if precision is not None: + self._wrapped = to_ctype(obj, precision=precision) + else: + self._wrapped = to_ctype(obj) def _init_c_obj(self): msg = "CAny is a generic adapter, not meant for initialization." @@ -111,11 +116,13 @@ def __repr__(self): @singledispatch -def to_ctype(value: Any): +def to_ctype(value: Any, *, precision: str | None = None): + _ = precision msg = f"No CType wrapper registered for {type(value)}" raise TypeError(msg) @to_ctype.register -def _(value: CType): +def _(value: CType, *, precision: str | None = None): + _ = precision return value diff --git a/vaccel/_c_types/wrappers/cbytes.py b/vaccel/_c_types/wrappers/cbytes.py index 23f98dd..51e33e8 100644 --- a/vaccel/_c_types/wrappers/cbytes.py +++ b/vaccel/_c_types/wrappers/cbytes.py @@ -85,6 +85,8 @@ def __eq__(self, other: "CBytes | bytes | bytearray | memoryview"): return self._data == other return NotImplemented + __hash__ = None + def __bytes__(self): return bytes(self._data) @@ -102,10 +104,12 @@ def __repr__(self): @to_ctype.register -def _(value: bytes): +def _(value: bytes, *, precision: str | None = None): + _ = precision return CBytes(value) @to_ctype.register -def _(value: bytearray): +def _(value: bytearray, *, precision: str | None = None): + _ = precision return CBytes(value) diff --git a/vaccel/_c_types/wrappers/cfloat.py b/vaccel/_c_types/wrappers/cfloat.py index d664293..85d6358 100644 --- a/vaccel/_c_types/wrappers/cfloat.py +++ b/vaccel/_c_types/wrappers/cfloat.py @@ -2,6 +2,8 @@ """C type interface for `float` objects.""" +from typing import Final + from vaccel._c_types.types import CType, to_ctype from vaccel._libvaccel import ffi from vaccel.error import NullPointerError @@ -22,6 +24,8 @@ class CFloat(CType): _ctype_str (str): The actual string that is used for the C type. """ + _SUPPORTED_PRECISIONS: Final[set[str]] = {"float", "double"} + def __init__(self, value: float, precision: str = "float"): """Initializes a new `CFloat` object. @@ -29,8 +33,9 @@ def __init__(self, value: float, precision: str = "float"): value: The float to be wrapped. precision: The C type that will be used to represent the float. """ - if precision not in ("float", "double"): - msg = "precision must be 'float' or 'double'" + if precision not in self._SUPPORTED_PRECISIONS: + supported = ", ".join(str(d) for d in self._SUPPORTED_PRECISIONS) + msg = f"Unsupported precision: {precision}. Supported: {supported}" raise ValueError(msg) self._value = float(value) self._precision = precision @@ -158,5 +163,5 @@ def __repr__(self): @to_ctype.register -def _(value: float): - return CFloat(value) +def _(value: float, *, precision: str | None = "float"): + return CFloat(value, precision) diff --git a/vaccel/_c_types/wrappers/cint.py b/vaccel/_c_types/wrappers/cint.py index a41c03e..e496620 100644 --- a/vaccel/_c_types/wrappers/cint.py +++ b/vaccel/_c_types/wrappers/cint.py @@ -2,6 +2,8 @@ """C type interface for `int` objects.""" +from typing import Final + from vaccel._c_types.types import CType, to_ctype from vaccel._libvaccel import ffi from vaccel.error import NullPointerError @@ -22,6 +24,20 @@ class CInt(CType): _ctype_str (str): The actual string that is used for the C type. """ + _SUPPORTED_PRECISIONS: Final[set[str]] = { + "int", + "int8_t", + "int16_t", + "int32_t", + "int64_t", + "unsigned", + "unsigned int", + "uint8_t", + "uint16_t", + "uint32_t", + "uint64_t", + } + def __init__(self, value: int, precision: str = "int"): """Initializes a new `CInt` object. @@ -29,9 +45,12 @@ def __init__(self, value: int, precision: str = "int"): value: The int to be wrapped. precision: The C type that will be used to represent the int. """ + if precision not in self._SUPPORTED_PRECISIONS: + supported = ", ".join(str(d) for d in self._SUPPORTED_PRECISIONS) + msg = f"Unsupported precision: {precision}. Supported: {supported}" + raise ValueError(msg) self._value = int(value) self._precision = precision - # TODO: validate user input # noqa: FIX002 self._ctype_str = self._precision super().__init__() @@ -156,5 +175,5 @@ def __repr__(self): @to_ctype.register -def _(value: int): - return CInt(value) +def _(value: int, *, precision: str | None = "int"): + return CInt(value, precision) diff --git a/vaccel/_c_types/wrappers/clist.py b/vaccel/_c_types/wrappers/clist.py index 07c53bc..f54525d 100644 --- a/vaccel/_c_types/wrappers/clist.py +++ b/vaccel/_c_types/wrappers/clist.py @@ -218,5 +218,6 @@ def __repr__(self): @to_ctype.register(tuple) @to_ctype.register(list) -def _(value: list): +def _(value: list, *, precision: str | None = None): + _ = precision return CList(value) diff --git a/vaccel/_c_types/wrappers/cnumpyarray.py b/vaccel/_c_types/wrappers/cnumpyarray.py index 6f91898..bf89df1 100644 --- a/vaccel/_c_types/wrappers/cnumpyarray.py +++ b/vaccel/_c_types/wrappers/cnumpyarray.py @@ -152,5 +152,6 @@ def is_contiguous(self) -> bool: if HAS_NUMPY: @to_ctype.register - def _(value: np.ndarray): + def _(value: np.ndarray, *, precision: str | None = None): + _ = precision return CNumpyArray(value) diff --git a/vaccel/_c_types/wrappers/cstr.py b/vaccel/_c_types/wrappers/cstr.py index 8b0fc9a..3a04d28 100644 --- a/vaccel/_c_types/wrappers/cstr.py +++ b/vaccel/_c_types/wrappers/cstr.py @@ -90,6 +90,8 @@ def __eq__(self, other: "CStr | str"): return self._value == other return NotImplemented + __hash__ = None + def __repr__(self): try: c_ptr = ( @@ -104,10 +106,12 @@ def __repr__(self): @to_ctype.register -def _(value: str): +def _(value: str, *, precision: str | None = None): + _ = precision return CStr(value) @to_ctype.register -def _(value: Path): +def _(value: Path, *, precision: str | None = None): + _ = precision return CStr(value) From 5309d9a229dd03e33480dafab5522a2538b5a0d8 Mon Sep 17 00:00:00 2001 From: Kostis Papazafeiropoulos Date: Fri, 31 Oct 2025 19:18:38 +0000 Subject: [PATCH 2/6] refactor(Arg): Update with new C API Update `Arg` with new C API; use new allocation/deallocation functions and implement arg type support PR: https://github.com/nubificus/vaccel-python/pull/29 Signed-off-by: Kostis Papazafeiropoulos Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- tests/test_genop.py | 35 +++++---- vaccel/__init__.py | 3 +- vaccel/arg.py | 175 +++++++++++++++++++++++++++++++++++++++++--- vaccel/ops/exec.py | 16 +++- 4 files changed, 198 insertions(+), 31 deletions(-) diff --git a/tests/test_genop.py b/tests/test_genop.py index 4ff2a7e..8918bf1 100644 --- a/tests/test_genop.py +++ b/tests/test_genop.py @@ -4,7 +4,7 @@ import pytest -from vaccel import Arg, OpType, Session +from vaccel import Arg, ArgType, OpType, Session @pytest.fixture @@ -41,9 +41,12 @@ def test_data(): def test_exec(test_lib, test_args): - arg_read = [OpType.EXEC, test_lib["path"], test_lib["symbol"]] - arg_read.extend(test_args["read"]) - g_arg_read = [Arg(arg) for arg in arg_read] + g_arg_read = [ + Arg(OpType.EXEC, ArgType.UINT8), + Arg(test_lib["path"], ArgType.STRING), + Arg(test_lib["symbol"], ArgType.STRING), + ] + g_arg_read += [Arg(arg) for arg in test_args["read"]] g_arg_write = [Arg(arg) for arg in test_args["write"]] session = Session() @@ -55,20 +58,20 @@ def test_exec(test_lib, test_args): def test_sgemm(test_data): arg_read = [ - Arg(OpType.BLAS_SGEMM), - Arg(test_data["m"]), - Arg(test_data["n"]), - Arg(test_data["k"]), - Arg(test_data["alpha"]), - Arg(test_data["a"]), - Arg(test_data["lda"]), - Arg(test_data["b"]), - Arg(test_data["ldb"]), - Arg(test_data["beta"]), - Arg(test_data["ldc"]), + Arg(OpType.BLAS_SGEMM, ArgType.UINT8), + Arg(test_data["m"], ArgType.INT64), + Arg(test_data["n"], ArgType.INT64), + Arg(test_data["k"], ArgType.INT64), + Arg(test_data["alpha"], ArgType.FLOAT32), + Arg(test_data["a"], ArgType.FLOAT32_ARRAY), + Arg(test_data["lda"], ArgType.INT64), + Arg(test_data["b"], ArgType.FLOAT32_ARRAY), + Arg(test_data["ldb"], ArgType.INT64), + Arg(test_data["beta"], ArgType.FLOAT32), + Arg(test_data["ldc"], ArgType.INT64), ] c = [float(0)] * test_data["m"] * test_data["n"] - arg_write = [Arg(c)] + arg_write = [Arg(c, ArgType.FLOAT32_ARRAY)] session = Session() session.genop(arg_read, arg_write) diff --git a/vaccel/__init__.py b/vaccel/__init__.py index b5368f5..8ab03a7 100644 --- a/vaccel/__init__.py +++ b/vaccel/__init__.py @@ -3,7 +3,7 @@ """Python API for vAccel.""" from ._version import __version__ -from .arg import Arg +from .arg import Arg, ArgType from .config import Config from .op import OpType from .resource import Resource, ResourceType @@ -12,6 +12,7 @@ __all__ = [ "Arg", + "ArgType", "Config", "OpType", "Resource", diff --git a/vaccel/arg.py b/vaccel/arg.py index 6c66b68..935a6f5 100644 --- a/vaccel/arg.py +++ b/vaccel/arg.py @@ -2,11 +2,106 @@ """Interface to the `struct vaccel_arg` C object.""" -from typing import Any +import logging +from typing import Any, Final from ._c_types import CAny, CType -from ._libvaccel import ffi -from .error import NullPointerError, ptr_or_raise +from ._c_types.utils import CEnumBuilder +from ._libvaccel import ffi, lib +from .error import FFIError, NullPointerError, ptr_or_raise + +logger = logging.getLogger(__name__) + +enum_builder = CEnumBuilder(lib) +ArgType = enum_builder.from_prefix("ArgType", "VACCEL_ARG_") + + +class ArgTypeMapper: + """Utility for mapping between `ArgType` and other common types.""" + + _NUMERIC_TYPES: Final[set[ArgType]] = { + ArgType.INT8, + ArgType.INT8_ARRAY, + ArgType.INT16, + ArgType.INT16_ARRAY, + ArgType.INT32, + ArgType.INT32_ARRAY, + ArgType.INT64, + ArgType.INT64_ARRAY, + ArgType.UINT8, + ArgType.UINT8_ARRAY, + ArgType.UINT16, + ArgType.UINT16_ARRAY, + ArgType.UINT32, + ArgType.UINT32_ARRAY, + ArgType.UINT64, + ArgType.UINT64_ARRAY, + ArgType.FLOAT32, + ArgType.FLOAT32_ARRAY, + ArgType.FLOAT64, + } + + _ARG_TYPE_TO_C: Final[dict[ArgType, str]] = { + ArgType.INT8: "int8_t", + ArgType.INT8_ARRAY: "int8_t *", + ArgType.INT16: "int16_t", + ArgType.INT16_ARRAY: "int16_t *", + ArgType.INT32: "int32_t", + ArgType.INT32_ARRAY: "int32_t *", + ArgType.INT64: "int64_t", + ArgType.INT64_ARRAY: "int64_t *", + ArgType.UINT8: "uint8_t", + ArgType.UINT8_ARRAY: "uint8_t *", + ArgType.UINT16: "uint16_t", + ArgType.UINT16_ARRAY: "uint16_t *", + ArgType.UINT32: "uint32_t", + ArgType.UINT32_ARRAY: "uint32_t *", + ArgType.UINT64: "uint64_t", + ArgType.UINT64_ARRAY: "uint64_t *", + ArgType.FLOAT32: "float", + ArgType.FLOAT32_ARRAY: "float *", + ArgType.FLOAT64: "double", + ArgType.FLOAT64_ARRAY: "double *", + ArgType.BOOL: "bool", + ArgType.BOOL_ARRAY: "bool *", + ArgType.CHAR: "char", + ArgType.CHAR_ARRAY: "char *", + ArgType.UCHAR: "unsigned char", + ArgType.UCHAR_ARRAY: "unsigned char *", + ArgType.STRING: "char *", + ArgType.BUFFER: "void *", + } + + @classmethod + def is_numeric(cls, arg_type: ArgType) -> bool: + """Checks if the arg type represents a numeric type. + + Args: + arg_type: The arg type value. + + Returns: + True if the arg type represents a numeric type. + """ + return arg_type in cls._NUMERIC_TYPES + + @classmethod + def type_to_c_type(cls, arg_type: ArgType) -> str: + """Converts an `ArgType` to a C type string. + + Args: + arg_type: The arg type value. + + Returns: + A corresponding C type as a string (e.g., "float", "int64_t"). + + Raises: + ValueError: If the `arg_type` value is not supported. + """ + if arg_type not in cls._ARG_TYPE_TO_C: + supported = ", ".join(str(d) for d in cls._ARG_TYPE_TO_C) + msg = f"Unsupported ArgType: {arg_type}. Supported: {supported}" + raise ValueError(msg) + return cls._ARG_TYPE_TO_C[arg_type] class Arg(CType): @@ -20,23 +115,49 @@ class Arg(CType): Attributes: _c_data (CAny): The encapsulated C data that is passed to the C struct. + _c_obj_ptr (ffi.CData): A double pointer to the underlying + `struct vaccel_arg` C object. + type_ (ArgType): The type of the arg. + custom_type_id (int): The user-specified type ID of the arg if the type + is `ArgType.CUSTOM`. """ - def __init__(self, data: Any): + def __init__( + self, data: Any, type_: ArgType = ArgType.RAW, custom_type_id: int = 0 + ): """Initializes a new `Arg` object. Args: data: The input data to be passed to the C struct. + type_: The type of the arg. + custom_type_id: The user-specified type ID of the arg if the type is + `ArgType.CUSTOM`. """ - self._c_data = CAny(data) + if ArgType != ArgType.RAW and ArgTypeMapper.is_numeric(type_): + precision = ArgTypeMapper.type_to_c_type(type_) + self._c_data = CAny(data, precision=precision) + else: + self._c_data = CAny(data) + self._c_obj_ptr = ffi.NULL + self._type = type_ + self._custom_type_id = custom_type_id super().__init__() def _init_c_obj(self): """Initializes the underlying `struct vaccel_arg` C object.""" - c_data = self._c_data - self._c_obj = ffi.new("struct vaccel_arg *") - self._c_obj.size = c_data.c_size - self._c_obj.buf = c_data._c_ptr + self._c_obj_ptr = ffi.new("struct vaccel_arg **") + ret = lib.vaccel_arg_from_buf( + self._c_obj_ptr, + self._c_data._c_ptr, + self._c_data.c_size, + self._type, + self._custom_type_id, + ) + if ret != 0: + raise FFIError(ret, "Could not initialize arg") + + self._c_obj = self._c_obj_ptr[0] + self._c_size = ffi.sizeof("struct vaccel_arg") @property def value(self) -> ffi.CData: @@ -47,6 +168,24 @@ def value(self) -> ffi.CData: """ return self._c_ptr_or_raise[0] + def _del_c_obj(self): + """Deletes the underlying `struct vaccel_arg` C object. + + Raises: + FFIError: If arg deletion fails. + """ + ret = lib.vaccel_arg_delete(self._c_ptr_or_raise) + if ret != 0: + raise FFIError(ret, "Could not delete arg") + + def __del__(self): + try: + self._del_c_obj() + except NullPointerError: + pass + except FFIError: + logger.exception("Failed to clean up Arg") + @property def buf(self) -> Any: """Returns the buffer value from the underlying C struct. @@ -62,6 +201,15 @@ def buf(self) -> Any: self._c_data, f"{self.__class__.__name__}._c_data" ).value + @property + def type(self) -> ArgType: + """The arg type. + + Returns: + The type of the arg. + """ + return ArgType(self._c_ptr_or_raise.type) + def __repr__(self): try: _c_ptr = ( @@ -70,6 +218,13 @@ def __repr__(self): else "NULL" ) size = self._c_obj.size if self._c_obj != ffi.NULL else 0 + type_ = self.type + type_name = getattr(type_, "name", repr(type_)) except (AttributeError, TypeError, NullPointerError): return f"<{self.__class__.__name__} (uninitialized or invalid)>" - return f"<{self.__class__.__name__} size={size} at {_c_ptr}>" + return ( + f"<{self.__class__.__name__} " + f"size={size} " + f"type={type_name} " + f"at {_c_ptr}>" + ) diff --git a/vaccel/ops/exec.py b/vaccel/ops/exec.py index ba772ef..8890c93 100644 --- a/vaccel/ops/exec.py +++ b/vaccel/ops/exec.py @@ -51,7 +51,9 @@ def exec( FFIError: If the C operation fails. """ if arg_read is not None: - c_arg_read = CList([Arg(arg) for arg in arg_read]) + c_arg_read = CList( + [arg if isinstance(arg, Arg) else Arg(arg) for arg in arg_read] + ) c_arg_read_ptr = c_arg_read._c_ptr c_arg_read_len = len(c_arg_read) else: @@ -60,7 +62,9 @@ def exec( c_arg_read_len = 0 if arg_write is not None: - c_arg_write = CList([Arg(arg) for arg in arg_write]) + c_arg_write = CList( + [arg if isinstance(arg, Arg) else Arg(arg) for arg in arg_write] + ) c_arg_write_ptr = c_arg_write._c_ptr c_arg_write_len = len(c_arg_write) else: @@ -112,7 +116,9 @@ def exec_with_resource( FFIError: If the C operation fails. """ if arg_read is not None: - c_arg_read = CList([Arg(arg) for arg in arg_read]) + c_arg_read = CList( + [arg if isinstance(arg, Arg) else Arg(arg) for arg in arg_read] + ) c_arg_read_ptr = c_arg_read._c_ptr c_arg_read_len = len(c_arg_read) else: @@ -121,7 +127,9 @@ def exec_with_resource( c_arg_read_len = 0 if arg_write is not None: - c_arg_write = CList([Arg(arg) for arg in arg_write]) + c_arg_write = CList( + [arg if isinstance(arg, Arg) else Arg(arg) for arg in arg_write] + ) c_arg_write_ptr = c_arg_write._c_ptr c_arg_write_len = len(c_arg_write) else: From 17e129f6d4cab9804a50c0acb7cb9a22dcd60120 Mon Sep 17 00:00:00 2001 From: Kostis Papazafeiropoulos Date: Fri, 31 Oct 2025 21:26:51 +0000 Subject: [PATCH 3/6] refactor(Session): Properly implement flags and update allocation Imlement `PluginType` enum for C enum `vaccel_plugin_type_t`, update `Session` class's `flags` type/methods and use the C heap allocation function PR: https://github.com/nubificus/vaccel-python/pull/29 Signed-off-by: Kostis Papazafeiropoulos Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- tests/test_general.py | 15 +++++++++++++-- vaccel/__init__.py | 2 ++ vaccel/_c_types/utils.py | 26 ++++++++++++++++++-------- vaccel/plugin.py | 11 +++++++++++ vaccel/session.py | 38 +++++++++++++++++++++++++++----------- 5 files changed, 71 insertions(+), 21 deletions(-) create mode 100644 vaccel/plugin.py diff --git a/tests/test_general.py b/tests/test_general.py index 83b0fb6..364407e 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -4,7 +4,7 @@ import pytest -from vaccel import Resource, ResourceType, Session +from vaccel import PluginType, Resource, ResourceType, Session @pytest.fixture @@ -13,9 +13,20 @@ def test_lib(vaccel_paths) -> Path: def test_session(): - ses_a = Session(flags=0) + ses_a = Session() + assert ses_a.id > 0 + assert ses_a.flags == 0 + assert ses_a.is_remote == 0 + ses_b = Session(flags=1) assert ses_b.id == ses_a.id + 1 + assert ses_b.flags == 1 + assert ses_b.is_remote == 0 + + ses_c = Session(flags=PluginType.GENERIC | PluginType.DEBUG) + assert ses_c.id == ses_b.id + 1 + assert ses_c.flags == PluginType.GENERIC | PluginType.DEBUG + assert ses_c.is_remote == 0 def test_resource(test_lib): diff --git a/vaccel/__init__.py b/vaccel/__init__.py index 8ab03a7..3beeba8 100644 --- a/vaccel/__init__.py +++ b/vaccel/__init__.py @@ -6,6 +6,7 @@ from .arg import Arg, ArgType from .config import Config from .op import OpType +from .plugin import PluginType from .resource import Resource, ResourceType from .session import Session from .vaccel import bootstrap, cleanup @@ -15,6 +16,7 @@ "ArgType", "Config", "OpType", + "PluginType", "Resource", "ResourceType", "Session", diff --git a/vaccel/_c_types/utils.py b/vaccel/_c_types/utils.py index bce1d4c..95afa57 100644 --- a/vaccel/_c_types/utils.py +++ b/vaccel/_c_types/utils.py @@ -2,7 +2,7 @@ """Utilities for C type conversions.""" -from enum import IntEnum +from enum import IntEnum, IntFlag from typing import Any @@ -26,21 +26,28 @@ def __init__(self, lib: Any): self.lib = lib self._cache = {} - def from_prefix(self, enum_name: str, prefix: str) -> IntEnum: + def from_prefix( + self, + enum_name: str, + prefix: str, + enum_type: type[IntEnum] | type[IntFlag] = IntEnum, + ) -> IntEnum | IntFlag: """Generates a Python enum from a C enum prefix. Dynamically create a Python `IntEnum` from C enum constants that share a prefix. Args: - enum_name: The name to give the `IntEnum`. + enum_name: The name to give the generated Enum. prefix: The prefix of the C constants (e.g., "VACCEL_"). + enum_type: The Enum base class to use. Returns: A Python IntEnum with values mapped from the C library. """ - if enum_name in self._cache: - return self._cache[enum_name] + cache_key = (enum_name, enum_type) + if cache_key in self._cache: + return self._cache[cache_key] members = { attr[len(prefix) :]: getattr(self.lib, attr) @@ -48,14 +55,17 @@ def from_prefix(self, enum_name: str, prefix: str) -> IntEnum: if attr.startswith(prefix) } + if not members: + msg = f"No constants found with prefix '{prefix}'" + raise ValueError(msg) + # Build docstring docstring = self._build_enum_docstring(enum_name, members) - # Generate enum - enum_cls = IntEnum(enum_name, members) + enum_cls = enum_type(enum_name, members) enum_cls.__doc__ = docstring - self._cache[enum_name] = enum_cls + self._cache[cache_key] = enum_cls return enum_cls def _build_enum_docstring( diff --git a/vaccel/plugin.py b/vaccel/plugin.py new file mode 100644 index 0000000..88428f5 --- /dev/null +++ b/vaccel/plugin.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Interface to the `struct vaccel_plugin` C object.""" + +from enum import IntFlag + +from ._c_types.utils import CEnumBuilder +from ._libvaccel import lib + +enum_builder = CEnumBuilder(lib) +PluginType = enum_builder.from_prefix("PluginType", "VACCEL_PLUGIN_", IntFlag) diff --git a/vaccel/session.py b/vaccel/session.py index 4ebcad3..c0f3927 100644 --- a/vaccel/session.py +++ b/vaccel/session.py @@ -17,6 +17,7 @@ from .ops.tf import TFMixin from .ops.tf.lite import TFLiteMixin from .ops.torch import TorchMixin +from .plugin import PluginType from .resource import Resource logger = logging.getLogger(__name__) @@ -32,16 +33,19 @@ class BaseSession(CType): CType: Abstract base class for defining C data types. Attributes: - _flags (int): The flags used to create the session. + _flags (PluginType): The flags used to create the session. + _c_obj_ptr (ffi.CData): A double pointer to the underlying + `struct vaccel_session` C object. """ - def __init__(self, flags: int = 0): + def __init__(self, flags: PluginType | int = 0): """Initializes a new `BaseSession` object. Args: flags: The flags to configure the session creation. Defaults to 0. """ - self._flags = flags + self._flags = PluginType(flags) + self._c_obj_ptr = ffi.NULL super().__init__() def _init_c_obj(self): @@ -50,12 +54,12 @@ def _init_c_obj(self): Raises: FFIError: If session initialization fails. """ - # TODO: Use vaccel_session_new() # noqa: FIX002 - self._c_obj = ffi.new("struct vaccel_session *") - ret = lib.vaccel_session_init(self._c_obj, self._flags) + self._c_obj_ptr = ffi.new("struct vaccel_session **") + ret = lib.vaccel_session_new(self._c_obj_ptr, self._flags) if ret != 0: raise FFIError(ret, "Could not init session") + self._c_obj = self._c_obj_ptr[0] self._c_size = ffi.sizeof("struct vaccel_session") @property @@ -73,7 +77,7 @@ def _del_c_obj(self): Raises: FFIError: If session release fails. """ - ret = lib.vaccel_session_release(self._c_ptr_or_raise) + ret = lib.vaccel_session_delete(self._c_ptr_or_raise) if ret != 0: raise FFIError(ret, "Could not release session") self._c_obj = ffi.NULL @@ -108,16 +112,26 @@ def remote_id(self) -> int: return int(self._c_ptr_or_raise.remote_id) @property - def flags(self) -> int: + def flags(self) -> PluginType: """The session flags. Returns: The flags set during session creation. """ - return int(self._c_ptr_or_raise.flags) + return PluginType(self._c_ptr_or_raise.hint) + + @property + def is_remote(self) -> bool: + """True if the session is remote. + + Returns: + True if the session is remote (the session plugin is a transport + plugin). + """ + return bool(self._c_ptr_or_raise.is_virtio) def has_resource(self, resource: Resource) -> bool: - """Check if a resource is registered with the session. + """Checks if a resource is registered with the session. Args: resource: The resource to check for registration. @@ -142,12 +156,14 @@ def __repr__(self): session_id = self.id remote_id = self.remote_id flags = self.flags + flags_name = getattr(flags, "name", repr(flags)) + flags_str = flags_name if flags_name is not None else f"0x{flags:x}" except (AttributeError, TypeError, NullPointerError): return f"<{self.__class__.__name__} (uninitialized or invalid)>" return ( f"<{self.__class__.__name__} id={session_id} " f"remote_id={remote_id} " - f"flags=0x{flags:x} " + f"flags={flags_str} " f"at {c_ptr}>" ) From 013e266a203abeb601e84826b462cb823146dddb Mon Sep 17 00:00:00 2001 From: Kostis Papazafeiropoulos Date: Fri, 31 Oct 2025 21:38:20 +0000 Subject: [PATCH 4/6] feat(Resource): Implement `*_from_buf()` and `*_sync` C operations Implement `*_from_buf()` for Python byte-like/numpy array objects and `*_sync` C operations in `Resource` class PR: https://github.com/nubificus/vaccel-python/pull/29 Signed-off-by: Kostis Papazafeiropoulos Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- tests/test_general.py | 17 +------ tests/test_resource.py | 103 +++++++++++++++++++++++++++++++++++++ vaccel/resource.py | 113 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 208 insertions(+), 25 deletions(-) create mode 100644 tests/test_resource.py diff --git a/tests/test_general.py b/tests/test_general.py index 364407e..b93ac8b 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -1,15 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -from pathlib import Path - -import pytest - -from vaccel import PluginType, Resource, ResourceType, Session - - -@pytest.fixture -def test_lib(vaccel_paths) -> Path: - return vaccel_paths["lib"] / "libmytestlib.so" +from vaccel import PluginType, Session def test_session(): @@ -29,12 +20,6 @@ def test_session(): assert ses_c.is_remote == 0 -def test_resource(test_lib): - res_a = Resource(test_lib, ResourceType.LIB) - res_b = Resource([test_lib, test_lib], ResourceType.LIB) - assert res_b.id == res_a.id + 1 - - def test_noop(): session = Session(flags=0) session.noop() diff --git a/tests/test_resource.py b/tests/test_resource.py new file mode 100644 index 0000000..dd3b0d9 --- /dev/null +++ b/tests/test_resource.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import numpy as np +import pytest + +from vaccel import Resource, ResourceType, Session +from vaccel._c_types import CBytes + + +@pytest.fixture +def test_lib(vaccel_paths) -> Path: + return vaccel_paths["lib"] / "libmytestlib.so" + + +@pytest.fixture +def test_buffer() -> bytes: + data = [1.0] * 30 + data_np = np.array(data, dtype=np.float32) + return { + "data": data, + "data_np": data_np, + "data_bytes": np.ascontiguousarray(data_np).tobytes(), + } + + +def test_resource(test_lib): + res_a = Resource(test_lib, ResourceType.LIB) + assert res_a.id > 0 + assert res_a.remote_id == 0 + + res_b = Resource([test_lib, test_lib], ResourceType.LIB) + assert res_b.id == res_a.id + 1 + assert res_b.remote_id == 0 + + +def test_resource_from_buffer(test_buffer): + res = Resource.from_buffer(test_buffer["data_bytes"], ResourceType.DATA) + res_data = res.value.blobs[0].data + res_size = res.value.blobs[0].size + assert res.id > 0 + assert res.remote_id == 0 + assert ( + CBytes.from_c_obj(res_data, res_size).value == test_buffer["data_bytes"] + ) + + +def test_resource_from_numpy(test_buffer): + res = Resource.from_numpy(test_buffer["data_np"]) + res_data = res.value.blobs[0].data + res_size = res.value.blobs[0].size + assert res.id > 0 + assert res.remote_id == 0 + assert ( + CBytes.from_c_obj(res_data, res_size).value == test_buffer["data_bytes"] + ) + + +def test_resource_register(test_lib): + res = Resource(test_lib, ResourceType.LIB) + ses = Session() + + res.register(ses) + assert res.is_registered(ses) + + +def test_resource_unregister(test_buffer): + res = Resource.from_buffer(test_buffer["data_bytes"], ResourceType.DATA) + ses = Session() + + res.register(ses) + res.unregister(ses) + assert not res.is_registered(ses) + + +def test_resource_register_unregister_multi(test_lib, test_buffer): + res_a = Resource.from_buffer(test_buffer["data_bytes"], ResourceType.DATA) + res_b = Resource(test_lib, ResourceType.LIB) + ses_a = Session() + ses_b = Session() + + res_a.register(ses_a) + assert res_a.is_registered(ses_a) + res_b.register(ses_a) + assert res_b.is_registered(ses_a) + + res_a.register(ses_b) + assert res_a.is_registered(ses_b) + res_b.register(ses_b) + assert res_b.is_registered(ses_b) + + res_a.sync(ses_a) + res_a.sync(ses_b) + + res_a.unregister(ses_a) + assert not res_a.is_registered(ses_a) + res_b.unregister(ses_a) + assert not res_b.is_registered(ses_a) + res_a.unregister(ses_b) + assert not res_a.is_registered(ses_b) + res_b.unregister(ses_b) + assert not res_b.is_registered(ses_b) diff --git a/vaccel/resource.py b/vaccel/resource.py index 1b6c020..78409fd 100644 --- a/vaccel/resource.py +++ b/vaccel/resource.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TYPE_CHECKING -from ._c_types import CList, CType +from ._c_types import CBytes, CList, CNumpyArray, CType from ._c_types.utils import CEnumBuilder from ._libvaccel import ffi, lib from .error import FFIError, NullPointerError @@ -16,6 +16,13 @@ BaseSession as Session, # Type hint only, not imported at runtime ) +try: + import numpy as np # noqa: TC002 + + HAS_NUMPY = True +except ImportError: + HAS_NUMPY = False + logger = logging.getLogger(__name__) enum_builder = CEnumBuilder(lib) @@ -35,6 +42,10 @@ class Resource(CType): _paths (list[Path] | list[str] | Path | str): The path(s) to the contained file(s). _type (ResourceType): The type of the resource. + _c_data (CBytes | CNumpyArray | None): The encapsulated buffer data + passed to the C struct. + _c_obj_ptr (ffi.CData): A double pointer to the underlying + `struct vaccel_resource` C object. """ def __init__( @@ -53,6 +64,8 @@ def __init__( self._paths = [str(paths)] self._c_paths = CList(self._paths) self._type = type_ + self._c_data = None + self._c_obj_ptr = ffi.NULL self.__sessions = [] super().__init__() @@ -63,18 +76,80 @@ def _init_c_obj(self): FFIError: If resource initialization fails. """ self._c_obj_ptr = ffi.new("struct vaccel_resource **") - ret = lib.vaccel_resource_multi_new( - self._c_obj_ptr, - self._c_paths._c_ptr, - len(self._c_paths), - self._type, - ) + if self._c_paths is not None: + ret = lib.vaccel_resource_multi_new( + self._c_obj_ptr, + self._c_paths._c_ptr, + len(self._c_paths), + self._type, + ) + else: + ret = lib.vaccel_resource_from_buf( + self._c_obj_ptr, + self._c_data._c_ptr, + self._c_data._c_size, + self._type, + ffi.NULL, + True, # noqa: FBT003 + ) if ret != 0: raise FFIError(ret, "Could not initialize resource") self._c_obj = self._c_obj_ptr[0] self._c_size = ffi.sizeof("struct vaccel_resource") + @classmethod + def from_buffer( + cls, + data: bytes | bytearray | memoryview, + type_: ResourceType, + ) -> "Resource": + """Initializes a new `Resource` object from byte-like data. + + Args: + data: The data to be passed to the C struct. + type_: The type of the resource. + + Returns: + A new `Resource` object + """ + inst = cls.__new__(cls) + inst._data = data + inst._type = type_ + inst._c_data = CBytes(inst._data) + inst._c_paths = None + inst._c_obj_ptr = ffi.NULL + inst.__sessions = [] + super().__init__(inst) + return inst + + @classmethod + def from_numpy(cls, data: "np.ndarray") -> "Resource": + """Initializes a new `Resource` object from a NumPy array. + + Args: + data: The NumPy array containing the resource data. + + Returns: + A new `Resource` object + + Raises: + NotImplementedError: If NumPy is not installed. + """ + if not HAS_NUMPY: + msg = "NumPy is not available" + raise NotImplementedError(msg) + + inst = cls.__new__(cls) + inst._data = data + inst._type = ResourceType.DATA + inst._c_data = CNumpyArray(inst._data) + inst._c_paths = None + inst._c_obj_ptr = ffi.NULL + inst.__sessions = [] + super().__init__(inst) + return inst + @property def value(self) -> ffi.CData: """Returns the value of the underlying C struct. @@ -140,7 +215,7 @@ def is_registered(self, session: "Session") -> bool: return session in self.__sessions def register(self, session: "Session") -> None: - """Register the resource with a session. + """Registers the resource with a session. Args: session: The session to register the resource with. @@ -161,7 +236,7 @@ def register(self, session: "Session") -> None: self.__sessions.append(session) def unregister(self, session: "Session") -> None: - """Unregister the resource from a session. + """Unregisters the resource from a session. Args: session: The session to unregister the resource from. @@ -181,6 +256,26 @@ def unregister(self, session: "Session") -> None: ) self.__sessions.remove(session) + def sync(self, session: "Session") -> None: + """Synchronizes the resource data to reflect any remote changes. + + Args: + session: The session the resource is registered with. + + Raises: + FFIError: If resource synchronization fails. + """ + ret = lib.vaccel_resource_sync( + self._c_ptr_or_raise, + session._c_ptr_or_raise, + ) + if ret != 0: + raise FFIError( + ret, + f"Could not synchronize resource {self.id} " + f"in session {session.id}", + ) + def __repr__(self): try: c_ptr = ( From 7959f56357dadc5961fc2c152afbca7429f11cc0 Mon Sep 17 00:00:00 2001 From: Kostis Papazafeiropoulos Date: Fri, 31 Oct 2025 21:43:22 +0000 Subject: [PATCH 5/6] test: Set library path from pkg-config file Set library path for tests from vAccel's pkg-config file instead of relying on the user to provide it PR: https://github.com/nubificus/vaccel-python/pull/29 Signed-off-by: Kostis Papazafeiropoulos Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- pyproject.toml | 2 +- tests/conftest.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7a419eb..b779568 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ ignore = [ convention = "google" [tool.ruff.lint.per-file-ignores] -"tests/**.py" = ["ANN001", "ANN201", "D", "INP001", "PLR2004", "S101", "S311"] +"tests/**.py" = ["ANN001", "ANN201", "D", "INP001", "PLR2004", "S101", "S311", "S603"] "examples/**.py" = ["ANN001", "ANN201", "D", "INP001", "T201"] "run-examples.py" = ["ANN001", "ANN201", "D", "INP001", "S603", "T201"] "build_ffi.py" = ["ANN001", "ANN201", "S603"] diff --git a/tests/conftest.py b/tests/conftest.py index 8815ceb..d1d277d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ import importlib.util import os +import subprocess +import sys from pathlib import Path import pkgconfig @@ -9,6 +11,8 @@ from build_ffi import compile_ffi +_REEXEC_MARKER = "_VACCEL_PYTEST_REEXEC_DONE" + def pytest_configure(): # Build ffi lib if package is not installed @@ -22,6 +26,23 @@ def pytest_configure(): os.environ["VACCEL_LOG_LEVEL"] = "4" +def pytest_cmdline_main(config): + _ = config + + # Set library path from pkgconfig + if _REEXEC_MARKER not in os.environ: + lib_path = os.environ.get("LD_LIBRARY_PATH", "") + vaccel_lib_path = pkgconfig.variables("vaccel")["libdir"] + + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ( + f"{vaccel_lib_path}:{lib_path}" if lib_path else vaccel_lib_path + ) + env[_REEXEC_MARKER] = "1" + + sys.exit(subprocess.call([sys.executable, *sys.argv], env=env)) + + @pytest.fixture(scope="session") def vaccel_paths(): variables = pkgconfig.variables("vaccel") From 5104cc7bb6da9e32bda01f764522a5e7f29c24a2 Mon Sep 17 00:00:00 2001 From: Kostis Papazafeiropoulos Date: Fri, 31 Oct 2025 21:45:39 +0000 Subject: [PATCH 6/6] fix(tf.Node): Correctly return node's ID Correctly access `vaccel_tf_node` C structure's ID in `tf.Node.id` method PR: https://github.com/nubificus/vaccel-python/pull/29 Signed-off-by: Kostis Papazafeiropoulos Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- vaccel/ops/tf/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vaccel/ops/tf/node.py b/vaccel/ops/tf/node.py index b498b76..43c953d 100644 --- a/vaccel/ops/tf/node.py +++ b/vaccel/ops/tf/node.py @@ -99,7 +99,7 @@ def id(self) -> int: Returns: The node's ID. """ - return int(self._c_ptr_or_raise.id[0]) + return int(self._c_ptr_or_raise.id) def __repr__(self): try: