From fcbf4c534fac7c82768f594d2a528c5adf25e663 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Sat, 7 Feb 2026 17:37:52 +1100 Subject: [PATCH] fix: `zstd` with sharding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `zstd` would try end decode the entire shard instead of a subchunk. The added test fails without the patch with: `ZarrV3Test.testShardingWithZstdCodecReadWrite:225 ยป Zstd Destination buffer is too small` --- .../zarrjava/v3/codec/core/ZstdCodec.java | 5 +++-- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index d7599bc..7d4b336 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -7,6 +7,7 @@ import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.codec.BytesBytesCodec; +import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.codec.Codec; @@ -28,7 +29,7 @@ public ZstdCodec( @Override public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { - byte[] compressedArray = compressedBytes.array(); + byte[] compressedArray = Utils.toArray(compressedBytes); long originalSize = Zstd.getFrameContentSize(compressedArray); if (originalSize == 0) { @@ -41,7 +42,7 @@ public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { @Override public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { - byte[] arr = chunkBytes.array(); + byte[] arr = Utils.toArray(chunkBytes); byte[] compressed; try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { ctx.setLevel(configuration.level); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index b48611a..601b6da 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -206,6 +206,27 @@ public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrExce Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); } + @Test + public void testShardingWithZstdCodecReadWrite() throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testShardingWithZstdCodecReadWrite"); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(8, 8, 8) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{2, 4, 8}, c1 -> c1.withZstd())); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + @Test public void testTransposeCodec() throws ZarrException { ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{