From 8d4a50d28454a229699b1bc1a05dfefca93bef34 Mon Sep 17 00:00:00 2001 From: theeldermillenial Date: Sat, 15 Mar 2025 21:33:19 -0400 Subject: [PATCH 1/6] Added IndefiniteDecoder for round trip plutusdata serialization --- pycardano/serialization.py | 31 +++++++++++++++++++++++----- test/pycardano/test_serialization.py | 27 +++++++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index c25139bc..8c309d44 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -2,6 +2,7 @@ from __future__ import annotations +from io import BytesIO import re import typing from collections import OrderedDict, UserList, defaultdict @@ -21,6 +22,7 @@ Iterable, List, Optional, + Sequence, Set, Type, TypeVar, @@ -41,6 +43,7 @@ pass from cbor2 import ( + CBORDecoder, CBOREncoder, CBORSimpleValue, CBORTag, @@ -199,6 +202,17 @@ def wrapper(cls, value: Primitive): CBORBase = TypeVar("CBORBase", bound="CBORSerializable") +class IndefiniteDecoder(CBORDecoder): + def decode_array(self, subtype: int) -> Sequence[Any]: + # Major tag 4 + length = self._decode_length(subtype, allow_indefinite=True) + + if length is None: + return IndefiniteList(super().decode_array(subtype=subtype)) + else: + return super().decode_array(subtype=subtype) + + def default_encoder( encoder: CBOREncoder, value: Union[CBORSerializable, IndefiniteList] ): @@ -265,7 +279,7 @@ class CBORSerializable: does not refer to itself, which could cause infinite loops. """ - def to_shallow_primitive(self) -> Primitive: + def to_shallow_primitive(self) -> Union[Primitive, CBORSerializable]: """ Convert the instance to a CBOR primitive. If the primitive is a container, e.g. list, dict, the type of its elements could be either a Primitive or a CBORSerializable. @@ -516,7 +530,10 @@ def from_cbor(cls, payload: Union[str, bytes]) -> CBORSerializable: """ if type(payload) is str: payload = bytes.fromhex(payload) - value = loads(payload) # type: ignore + + with BytesIO(payload) as fp: + value = IndefiniteDecoder(fp).decode() + return cls.from_primitive(value) def __repr__(self): @@ -580,10 +597,14 @@ def _restore_typed_primitive( raise DeserializeException( f"List types need exactly one type argument, but got {t_args}" ) - t = t_args[0] - if not isinstance(v, list): + t_subtype = t_args[0] + if not isinstance(v, (list, IndefiniteList)): raise DeserializeException(f"Expected type list but got {type(v)}") - return IndefiniteList([_restore_typed_primitive(t, w) for w in v]) + v_list = [_restore_typed_primitive(t_subtype, w) for w in v] + if t == IndefiniteList: + return IndefiniteList(v_list) + else: + return v_list elif isclass(t) and t == ByteString: if not isinstance(v, bytes): raise DeserializeException(f"Expected type bytes but got {type(v)}") diff --git a/test/pycardano/test_serialization.py b/test/pycardano/test_serialization.py index 9deb2233..f2faaa5e 100644 --- a/test/pycardano/test_serialization.py +++ b/test/pycardano/test_serialization.py @@ -30,7 +30,7 @@ VerificationKeyWitness, ) from pycardano.exception import DeserializeException, SerializeException -from pycardano.plutus import PlutusV1Script, PlutusV2Script +from pycardano.plutus import PlutusData, PlutusV1Script, PlutusV2Script from pycardano.serialization import ( ArrayCBORSerializable, ByteString, @@ -368,6 +368,31 @@ class Test1(CBORSerializable): obj.validate() +@pytest.mark.xfail +def test_datum_raw_round_trip(): + @dataclass + class TestDatum(PlutusData): + CONSTR_ID = 0 + a: int + b: List[bytes] + + datum = TestDatum(a=1, b=[b"test", b"datum"]) + restored = RawPlutusData.from_cbor(datum.to_cbor()) + assert datum.to_cbor_hex() == restored.to_cbor_hex() + + +def test_datum_round_trip(): + @dataclass + class TestDatum(PlutusData): + CONSTR_ID = 0 + a: int + b: List[bytes] + + datum = TestDatum(a=1, b=[b"test", b"datum"]) + restored = TestDatum.from_cbor(datum.to_cbor()) + assert datum.to_cbor_hex() == restored.to_cbor_hex() + + def test_wrong_primitive_type(): @dataclass class Test1(MapCBORSerializable): From d0fbc02a51687328cd77066200d52c208706a303 Mon Sep 17 00:00:00 2001 From: theeldermillenial Date: Sat, 15 Mar 2025 21:54:04 -0400 Subject: [PATCH 2/6] Fixed qa issues --- pycardano/serialization.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 8c309d44..9f954d79 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -49,7 +49,6 @@ CBORTag, FrozenDict, dumps, - loads, undefined, ) from frozenlist import FrozenList @@ -208,7 +207,9 @@ def decode_array(self, subtype: int) -> Sequence[Any]: length = self._decode_length(subtype, allow_indefinite=True) if length is None: - return IndefiniteList(super().decode_array(subtype=subtype)) + return IndefiniteList( + cast(Primitive, super().decode_array(subtype=subtype)) + ) else: return super().decode_array(subtype=subtype) @@ -531,7 +532,7 @@ def from_cbor(cls, payload: Union[str, bytes]) -> CBORSerializable: if type(payload) is str: payload = bytes.fromhex(payload) - with BytesIO(payload) as fp: + with BytesIO(cast(bytes, payload)) as fp: value = IndefiniteDecoder(fp).decode() return cls.from_primitive(value) @@ -555,7 +556,7 @@ def _restore_dataclass_field( if "object_hook" in f.metadata: return f.metadata["object_hook"](v) - return _restore_typed_primitive(f.type, v) + return _restore_typed_primitive(cast(Any, f.type), v) def _restore_typed_primitive( From 81961acdd16b08c83bf4e855d67f1a9c31a7df54 Mon Sep 17 00:00:00 2001 From: Jerry Date: Sun, 23 Mar 2025 10:46:11 -0700 Subject: [PATCH 3/6] Force non-binary installation of cbor2 --- .gitignore | 1 + Makefile | 22 ++++++++++++++++++---- pycardano/serialization.py | 27 ++++++++++++++------------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 82095d0a..c3dbfbd6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ docs/build dist .mypy_cache coverage.xml +.cbor2_version # IDE .idea diff --git a/Makefile b/Makefile index 401813cd..025ecb81 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,24 @@ export PRINT_HELP_PYSCRIPT BROWSER := poetry run python -c "$$BROWSER_PYSCRIPT" +ensure-pure-cbor2: ## ensures cbor2 is installed with pure Python implementation + @poetry run python -c "from importlib.metadata import version; \ + print(version('cbor2'))" > .cbor2_version + @poetry run python -c "import cbor2, inspect; \ + print('Checking cbor2 implementation...'); \ + decoder_path = inspect.getfile(cbor2.CBORDecoder); \ + using_c_ext = decoder_path.endswith('.so'); \ + print(f'Implementation path: {decoder_path}'); \ + print(f'Using C extension: {using_c_ext}'); \ + exit(1 if using_c_ext else 0)" || \ + (echo "Reinstalling cbor2 with pure Python implementation..." && \ + poetry run pip install --no-binary cbor2 "cbor2==$$(cat .cbor2_version)" --force-reinstall && \ + rm .cbor2_version) + help: @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) -cov: ## check code coverage +cov: ensure-pure-cbor2 ## check code coverage poetry run pytest -n 4 --cov pycardano cov-html: cov ## check code coverage and generate an html report @@ -54,7 +68,7 @@ clean-test: ## remove test and coverage artifacts rm -fr cov_html/ rm -fr .pytest_cache -test: ## runs tests +test: ensure-pure-cbor2 ## runs tests poetry run pytest -vv -n 4 test-integration: ## runs integration tests @@ -63,7 +77,7 @@ test-integration: ## runs integration tests test-single: ## runs tests with "single" markers poetry run pytest -s -vv -m single -qa: ## runs static analyses +qa: ensure-pure-cbor2 ## runs static analyses poetry run flake8 pycardano poetry run mypy --install-types --non-interactive pycardano poetry run black --check . @@ -78,6 +92,6 @@ docs: ## build the documentation poetry run sphinx-build docs/source docs/build/html $(BROWSER) docs/build/html/index.html -release: clean qa test format ## build dist version and release to pypi +release: clean qa test format ensure-pure-cbor2 ## build dist version and release to pypi poetry build poetry publish \ No newline at end of file diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 9f954d79..9572732a 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -200,19 +200,21 @@ def wrapper(cls, value: Primitive): CBORBase = TypeVar("CBORBase", bound="CBORSerializable") +def decode_array(self, subtype: int) -> Sequence[Any]: + # Major tag 4 + length = self._decode_length(subtype, allow_indefinite=True) -class IndefiniteDecoder(CBORDecoder): - def decode_array(self, subtype: int) -> Sequence[Any]: - # Major tag 4 - length = self._decode_length(subtype, allow_indefinite=True) - - if length is None: - return IndefiniteList( - cast(Primitive, super().decode_array(subtype=subtype)) - ) - else: - return super().decode_array(subtype=subtype) + if length is None: + return IndefiniteList( + cast(Primitive, self.decode_array(subtype=subtype)) + ) + else: + return self.decode_array(subtype=subtype) +try: + cbor2._decoder.major_decoders[4] = decode_array +except Exception as e: + logger.warning("Failed to replace major decoder for indefinite array", e) def default_encoder( encoder: CBOREncoder, value: Union[CBORSerializable, IndefiniteList] @@ -532,8 +534,7 @@ def from_cbor(cls, payload: Union[str, bytes]) -> CBORSerializable: if type(payload) is str: payload = bytes.fromhex(payload) - with BytesIO(cast(bytes, payload)) as fp: - value = IndefiniteDecoder(fp).decode() + value = cbor2.loads(payload) return cls.from_primitive(value) From 9f457c4a5b9e62d642be6a19349caa3e2999134c Mon Sep 17 00:00:00 2001 From: Jerry Date: Sun, 23 Mar 2025 10:56:40 -0700 Subject: [PATCH 4/6] Ensure pure cbor2 in CI --- .github/workflows/main.yml | 3 +++ .github/workflows/publish.yml | 3 +++ Makefile | 3 ++- integration-test/run_tests.sh | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b01d2d6..f73009fa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,6 +28,9 @@ jobs: - name: Install dependencies run: | poetry install + - name: Ensure pure cbor2 is installed + run: | + make ensure-pure-cbor2 - name: Run unit tests run: | poetry run pytest --doctest-modules --ignore=examples --cov=pycardano --cov-config=.coveragerc --cov-report=xml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 02c530ba..b1f8a9e7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,6 +24,9 @@ jobs: - name: Install dependencies run: | poetry install + - name: Ensure pure cbor2 is installed + run: | + make ensure-pure-cbor2 - name: Lint with flake8 run: | poetry run flake8 pycardano diff --git a/Makefile b/Makefile index 025ecb81..f7b8fcec 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,8 @@ ensure-pure-cbor2: ## ensures cbor2 is installed with pure Python implementation print(f'Using C extension: {using_c_ext}'); \ exit(1 if using_c_ext else 0)" || \ (echo "Reinstalling cbor2 with pure Python implementation..." && \ - poetry run pip install --no-binary cbor2 "cbor2==$$(cat .cbor2_version)" --force-reinstall && \ + poetry run pip uninstall -y cbor2 && \ + CBOR2_BUILD_C_EXTENSION=0 poetry run pip install --no-binary cbor2 "cbor2==$$(cat .cbor2_version)" --force-reinstall && \ rm .cbor2_version) help: diff --git a/integration-test/run_tests.sh b/integration-test/run_tests.sh index b9119b15..cc5cccf9 100755 --- a/integration-test/run_tests.sh +++ b/integration-test/run_tests.sh @@ -6,6 +6,7 @@ set -o pipefail ROOT=$(pwd) poetry install -C .. +make ensure-pure-cbor2 -f ../Makefile #poetry run pip install ogmios ########## From 6f5b64b390eb2f91fb92a49d1527664b41798718 Mon Sep 17 00:00:00 2001 From: theeldermillenial Date: Sun, 23 Mar 2025 19:47:15 -0400 Subject: [PATCH 5/6] Added IndefiniteList to ArrayCBORSerializable primitives --- pycardano/serialization.py | 4 ++-- test/pycardano/test_serialization.py | 1 - test/pycardano/test_transaction.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 9572732a..2f638b0a 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -735,8 +735,8 @@ def to_shallow_primitive(self) -> Primitive: return primitives @classmethod - @limit_primitive_type(list, tuple) - def from_primitive(cls: Type[ArrayBase], values: Union[list, tuple]) -> ArrayBase: + @limit_primitive_type(list, tuple, IndefiniteList) + def from_primitive(cls: Type[ArrayBase], values: Union[list, tuple, IndefiniteList]) -> ArrayBase: """Restore a primitive value to its original class type. Args: diff --git a/test/pycardano/test_serialization.py b/test/pycardano/test_serialization.py index f2faaa5e..81fa12fd 100644 --- a/test/pycardano/test_serialization.py +++ b/test/pycardano/test_serialization.py @@ -368,7 +368,6 @@ class Test1(CBORSerializable): obj.validate() -@pytest.mark.xfail def test_datum_raw_round_trip(): @dataclass class TestDatum(PlutusData): diff --git a/test/pycardano/test_transaction.py b/test/pycardano/test_transaction.py index a7f5022d..59c0b66f 100644 --- a/test/pycardano/test_transaction.py +++ b/test/pycardano/test_transaction.py @@ -416,6 +416,21 @@ def test_multi_asset_comparison(): with pytest.raises(TypeCheckError): a <= 1 +def test_datum_witness(): + @dataclass + class TestDatum(PlutusData): + CONSTR_ID = 0 + a: int + b: bytes + + tx_body = make_transaction_body() + signed_tx = Transaction( + tx_body, + TransactionWitnessSet(vkey_witnesses=None, plutus_data=[TestDatum(1, b"test")]), + ) + restored_tx = Transaction.from_cbor(signed_tx.to_cbor()) + + assert signed_tx.to_cbor_hex() == restored_tx.to_cbor_hex() def test_values(): a = Value.from_primitive( From a243faa2ed9d14f16c9cfdad0d2ae93aaaddcd0e Mon Sep 17 00:00:00 2001 From: theeldermillenial Date: Sun, 23 Mar 2025 19:53:32 -0400 Subject: [PATCH 6/6] Pass QA checks --- pycardano/serialization.py | 15 +++++++++------ test/pycardano/test_transaction.py | 2 ++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 2f638b0a..b2c888d1 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -2,7 +2,6 @@ from __future__ import annotations -from io import BytesIO import re import typing from collections import OrderedDict, UserList, defaultdict @@ -43,7 +42,6 @@ pass from cbor2 import ( - CBORDecoder, CBOREncoder, CBORSimpleValue, CBORTag, @@ -200,22 +198,23 @@ def wrapper(cls, value: Primitive): CBORBase = TypeVar("CBORBase", bound="CBORSerializable") + def decode_array(self, subtype: int) -> Sequence[Any]: # Major tag 4 length = self._decode_length(subtype, allow_indefinite=True) if length is None: - return IndefiniteList( - cast(Primitive, self.decode_array(subtype=subtype)) - ) + return IndefiniteList(cast(Primitive, self.decode_array(subtype=subtype))) else: return self.decode_array(subtype=subtype) + try: cbor2._decoder.major_decoders[4] = decode_array except Exception as e: logger.warning("Failed to replace major decoder for indefinite array", e) + def default_encoder( encoder: CBOREncoder, value: Union[CBORSerializable, IndefiniteList] ): @@ -534,6 +533,8 @@ def from_cbor(cls, payload: Union[str, bytes]) -> CBORSerializable: if type(payload) is str: payload = bytes.fromhex(payload) + assert isinstance(payload, bytes) + value = cbor2.loads(payload) return cls.from_primitive(value) @@ -736,7 +737,9 @@ def to_shallow_primitive(self) -> Primitive: @classmethod @limit_primitive_type(list, tuple, IndefiniteList) - def from_primitive(cls: Type[ArrayBase], values: Union[list, tuple, IndefiniteList]) -> ArrayBase: + def from_primitive( + cls: Type[ArrayBase], values: Union[list, tuple, IndefiniteList] + ) -> ArrayBase: """Restore a primitive value to its original class type. Args: diff --git a/test/pycardano/test_transaction.py b/test/pycardano/test_transaction.py index 59c0b66f..62796ae2 100644 --- a/test/pycardano/test_transaction.py +++ b/test/pycardano/test_transaction.py @@ -416,6 +416,7 @@ def test_multi_asset_comparison(): with pytest.raises(TypeCheckError): a <= 1 + def test_datum_witness(): @dataclass class TestDatum(PlutusData): @@ -432,6 +433,7 @@ class TestDatum(PlutusData): assert signed_tx.to_cbor_hex() == restored_tx.to_cbor_hex() + def test_values(): a = Value.from_primitive( [1, {b"1" * SCRIPT_HASH_SIZE: {b"Token1": 1, b"Token2": 2}}]