From 9fe9467f31ef3ff900711f439dceb3c34f2e0b0d Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 20:08:05 -0800 Subject: [PATCH 01/10] upgrade to iceberg 1.10.1 --- demo/notebook/demo.ipynb | 2 +- pom.xml | 2 +- xtable-aws/src/main/resources/META-INF/LICENSE-bundled | 4 ++-- .../src/main/resources/META-INF/LICENSE-bundled | 4 ++-- .../src/main/resources/META-INF/LICENSE-bundled | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/demo/notebook/demo.ipynb b/demo/notebook/demo.ipynb index 090d67d09..b208c9250 100644 --- a/demo/notebook/demo.ipynb +++ b/demo/notebook/demo.ipynb @@ -30,7 +30,7 @@ "import $cp.`/home/jars/xtable-core-0.2.0-SNAPSHOT.jar`\n", "import $cp.`/home/jars/xtable-api-0.2.0-SNAPSHOT.jar`\n", "import $cp.`/home/jars/xtable-hudi-support-utils-0.2.0-SNAPSHOT.jar`\n", - "import $ivy.`org.apache.iceberg:iceberg-hive-runtime:1.4.2`\n", + "import $ivy.`org.apache.iceberg:iceberg-hive-runtime:1.10.1`\n", "import $ivy.`io.trino:trino-jdbc:431`\n", "import java.util._\n", "import org.apache.xtable.conversion._\n", diff --git a/pom.xml b/pom.xml index 855894ec2..43b76955c 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ 2.12 3.4.2 3.4 - 1.4.2 + 1.10.1 2.4.0 1.2.0 2.18.2 diff --git a/xtable-aws/src/main/resources/META-INF/LICENSE-bundled b/xtable-aws/src/main/resources/META-INF/LICENSE-bundled index c73a8939c..70a583c08 100644 --- a/xtable-aws/src/main/resources/META-INF/LICENSE-bundled +++ b/xtable-aws/src/main/resources/META-INF/LICENSE-bundled @@ -226,8 +226,8 @@ org.apache.httpcomponents:fluent-hc:4.4.1 org.apache.hudi:hudi-common:0.14.0 org.apache.hudi:hudi-java-client:0.14.0 org.apache.hudi:hudi-timeline-service:0.14.0 -org.apache.iceberg:iceberg-api:1.4.2 -org.apache.iceberg:iceberg-core:1.4.2 +org.apache.iceberg:iceberg-api:1.10.1 +org.apache.iceberg:iceberg-core:1.10.1 org.apache.logging.log4j:log4j-1.2-api:2.22.0 org.apache.logging.log4j:log4j-api:2.22.0 org.javassist:javassist:3.25.0-GA diff --git a/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled b/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled index 04fe28aa5..27c286a32 100644 --- a/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled +++ b/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled @@ -225,8 +225,8 @@ org.apache.httpcomponents:fluent-hc:4.4.1 org.apache.hudi:hudi-common:0.14.0 org.apache.hudi:hudi-java-client:0.14.0 org.apache.hudi:hudi-timeline-service:0.14.0 -org.apache.iceberg:iceberg-api:1.4.2 -org.apache.iceberg:iceberg-core:1.4.2 +org.apache.iceberg:iceberg-api:1.10.1 +org.apache.iceberg:iceberg-core:1.10.1 org.apache.logging.log4j:log4j-1.2-api:2.22.0 org.apache.logging.log4j:log4j-api:2.22.0 io.delta:delta-core_2.12:2.4.0 diff --git a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled index 46089bf38..de1b2a89a 100644 --- a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled +++ b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled @@ -233,9 +233,9 @@ org.apache.hudi:hudi-client-common:0.14.0 org.apache.hudi:hudi-common:0.14.0 org.apache.hudi:hudi-timeline-service:0.14.0 org.apache.hudi:hudi-sync-common:0.14.0 -org.apache.iceberg:iceberg-api:1.4.2 -org.apache.iceberg:iceberg-core:1.4.2 -org.apache.iceberg:iceberg-parquet:1.4.2 +org.apache.iceberg:iceberg-api:1.10.1 +org.apache.iceberg:iceberg-core:1.10.1 +org.apache.iceberg:iceberg-parquet:1.10.1 org.apache.logging.log4j:log4j-api:2.22.0 org.apache.logging.log4j:log4j-1.2-api:2.22.0 org.roaringbitmap:RoaringBitmap:0.9.47 From 41142e2e74644e8866610938a2f2e76e67486a85 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 20:49:58 -0800 Subject: [PATCH 02/10] api changes --- .../src/test/java/org/apache/xtable/TestIcebergTable.java | 2 +- .../org/apache/xtable/iceberg/TestIcebergConversionSource.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java index b7b86578c..fe1414574 100644 --- a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java +++ b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java @@ -371,7 +371,7 @@ private DataFile writeAndGetDataFile(List records, StructLike partitionK DataWriter dataWriter = Parquet.writeData(file) .schema(icebergTable.schema()) - .createWriterFunc(GenericParquetWriter::buildWriter) + .createWriterFunc(GenericParquetWriter::create) .overwrite() .withSpec(icebergTable.spec()) .withPartition(partitionKey) diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java index d1bf70bb6..10372e0b4 100644 --- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java +++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java @@ -432,7 +432,7 @@ private DataFile generateTestDataFile(int partition, Table table, String filePat DataWriter dataWriter = Parquet.writeData(table.io().newOutputFile(filePath)) .schema(csSchema) - .createWriterFunc(GenericParquetWriter::buildWriter) + .createWriterFunc(GenericParquetWriter::create) .overwrite() .withSpec(table.spec()) .withPartition(partitionInfo) From cb4405d1f0d5ac087edc0a42838ef6e5e4a0bff8 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 20:50:18 -0800 Subject: [PATCH 03/10] use best available iceberg-hive-runtime --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 43b76955c..888bd16cd 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ 3.4.2 3.4 1.10.1 + 1.7.2 2.4.0 1.2.0 2.18.2 @@ -315,7 +316,7 @@ org.apache.iceberg iceberg-hive-runtime - ${iceberg.version} + ${iceberg.hive.runtime.version} From 04fa576d1e35ed6a20b45654ae0adcba4ba3776f Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 21:29:59 -0800 Subject: [PATCH 04/10] upgrade avro version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 888bd16cd..5bc2d11f4 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 2025-01-01T00:00:00Z 8 3.13.0 - 1.11.4 + 1.12.0 2.22.0 5.11.4 1.11.4 From c04f9ce19d8f6f237e69bf63732b53c3c3f02984 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 21:30:07 -0800 Subject: [PATCH 05/10] to run with java 17 --- pom.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pom.xml b/pom.xml index 5bc2d11f4..3c9fdd72e 100644 --- a/pom.xml +++ b/pom.xml @@ -749,6 +749,22 @@ true false 120 + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED + From a2d6484302dbcb806f45842cf7f3217d86cca0fc Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 21:30:15 -0800 Subject: [PATCH 06/10] tests --- .../java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java index b07fac4c5..2849c5a67 100644 --- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java +++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java @@ -181,7 +181,7 @@ public void testUpdateListElement() { verify(mockUpdateSchema).deleteColumn("array_field.element.element_string"); verify(mockUpdateSchema) - .addColumn("array_field.element", "element_double", Types.DoubleType.get(), null); + .addColumn("array_field.element", "element_double", Types.DoubleType.get(), (String) null); verify(mockUpdateSchema).commit(); } @@ -194,7 +194,7 @@ public void testUpdateMapValue() { verify(mockUpdateSchema).deleteColumn("map_field.value.value_string"); verify(mockUpdateSchema) - .addRequiredColumn("map_field.value", "value_double", Types.DoubleType.get(), null); + .addRequiredColumn("map_field.value", "value_double", Types.DoubleType.get(), (String) null); verify(mockUpdateSchema).commit(); } From 34f3a075e78dfe656275550d5a69cce64432517b Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 22:12:34 -0800 Subject: [PATCH 07/10] fix avro --- xtable-core/src/test/resources/schemas/basic_schema.avsc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtable-core/src/test/resources/schemas/basic_schema.avsc b/xtable-core/src/test/resources/schemas/basic_schema.avsc index 613094c12..8a317c51b 100644 --- a/xtable-core/src/test/resources/schemas/basic_schema.avsc +++ b/xtable-core/src/test/resources/schemas/basic_schema.avsc @@ -72,8 +72,8 @@ }, { "name": "bytes_field", - "type": "bytes", - "default": "" + "type": ["null", "bytes"], + "default": null }, { "name": "decimal_field", From 89247c980d82cf5b830f60bff1ad369b52614736 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 22:18:03 -0800 Subject: [PATCH 08/10] dont change notebook --- demo/notebook/demo.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/notebook/demo.ipynb b/demo/notebook/demo.ipynb index b208c9250..090d67d09 100644 --- a/demo/notebook/demo.ipynb +++ b/demo/notebook/demo.ipynb @@ -30,7 +30,7 @@ "import $cp.`/home/jars/xtable-core-0.2.0-SNAPSHOT.jar`\n", "import $cp.`/home/jars/xtable-api-0.2.0-SNAPSHOT.jar`\n", "import $cp.`/home/jars/xtable-hudi-support-utils-0.2.0-SNAPSHOT.jar`\n", - "import $ivy.`org.apache.iceberg:iceberg-hive-runtime:1.10.1`\n", + "import $ivy.`org.apache.iceberg:iceberg-hive-runtime:1.4.2`\n", "import $ivy.`io.trino:trino-jdbc:431`\n", "import java.util._\n", "import org.apache.xtable.conversion._\n", From d3f5674195ed233e7cc3b47ae6abd1c65c67d839 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 14 Jan 2026 22:52:39 -0800 Subject: [PATCH 09/10] fix tests --- pom.xml | 18 +++++++++++++++- .../xtable/hudi/HudiFileStatsExtractor.java | 20 +++++++++++------- .../kernel/DeltaKernelSchemaExtractor.java | 5 ++--- .../xtable/iceberg/TestIcebergDataHelper.java | 6 +----- .../xtable/iceberg/TestIcebergSchemaSync.java | 3 ++- .../test/resources/schemas/basic_schema.avsc | 21 +++++++++++-------- 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index 3c9fdd72e..391a00487 100644 --- a/pom.xml +++ b/pom.xml @@ -784,7 +784,23 @@ false 6 false - -Xmx1500m + + -Xmx1500m + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED + 120 diff --git a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java index 82a094938..5a0b70cb9 100644 --- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java +++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java @@ -203,14 +203,18 @@ private static ColumnStat getColumnStatFromHudiStat( if (field.getSchema().getDataType() == InternalType.DECIMAL) { int scale = (int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_SCALE); - minValue = - minValue instanceof ByteBuffer - ? convertBytesToBigDecimal((ByteBuffer) minValue, scale) - : ((BigDecimal) minValue).setScale(scale, RoundingMode.UNNECESSARY); - maxValue = - maxValue instanceof ByteBuffer - ? convertBytesToBigDecimal((ByteBuffer) maxValue, scale) - : ((BigDecimal) maxValue).setScale(scale, RoundingMode.UNNECESSARY); + if (minValue != null) { + minValue = + minValue instanceof ByteBuffer + ? convertBytesToBigDecimal((ByteBuffer) minValue, scale) + : ((BigDecimal) minValue).setScale(scale, RoundingMode.UNNECESSARY); + } + if (maxValue != null) { + maxValue = + maxValue instanceof ByteBuffer + ? convertBytesToBigDecimal((ByteBuffer) maxValue, scale) + : ((BigDecimal) maxValue).setScale(scale, RoundingMode.UNNECESSARY); + } } return getColumnStatFromValues( minValue, diff --git a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java index e3da2e7d2..fccd5b22f 100644 --- a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java +++ b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java @@ -66,9 +66,6 @@ public InternalSchema toInternalSchema(StructType structType) { return toInternalSchema(structType, null, false, null, null); } - String trimmedTypeName = ""; - InternalType type = null; - private InternalSchema toInternalSchema( DataType dataType, String parentPath, @@ -76,6 +73,8 @@ private InternalSchema toInternalSchema( String comment, FieldMetadata originalMetadata) { + String trimmedTypeName = ""; + InternalType type = null; Map metadata = null; List fields = null; diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java index a8518c1f9..d53b72067 100644 --- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java +++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java @@ -333,11 +333,7 @@ private Object generateRandomValueForType( case STRUCT: return generateInsertRecord(timeLowerBound, timeUpperBound, fieldType.asStructType()); case UUID: - UUID uuid = UUID.randomUUID(); - ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]); - byteBuffer.putLong(uuid.getMostSignificantBits()); - byteBuffer.putLong(uuid.getLeastSignificantBits()); - return byteBuffer.array(); + return UUID.randomUUID(); case LIST: Types.ListType listType = (Types.ListType) fieldType; int listSize = RANDOM.nextInt(5) + 1; diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java index 2849c5a67..ffdd5f996 100644 --- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java +++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java @@ -194,7 +194,8 @@ public void testUpdateMapValue() { verify(mockUpdateSchema).deleteColumn("map_field.value.value_string"); verify(mockUpdateSchema) - .addRequiredColumn("map_field.value", "value_double", Types.DoubleType.get(), (String) null); + .addRequiredColumn( + "map_field.value", "value_double", Types.DoubleType.get(), (String) null); verify(mockUpdateSchema).commit(); } diff --git a/xtable-core/src/test/resources/schemas/basic_schema.avsc b/xtable-core/src/test/resources/schemas/basic_schema.avsc index 8a317c51b..37193be59 100644 --- a/xtable-core/src/test/resources/schemas/basic_schema.avsc +++ b/xtable-core/src/test/resources/schemas/basic_schema.avsc @@ -77,15 +77,18 @@ }, { "name": "decimal_field", - "type": { - "type": "fixed", - "name": "decimal_field_testing", - "size": 10, - "logicalType": "decimal", - "precision": 20, - "scale": 2 - }, - "default": "\u0000" + "type": [ + "null", + { + "type": "fixed", + "name": "decimal_field_testing", + "size": 10, + "logicalType": "decimal", + "precision": 20, + "scale": 2 + } + ], + "default": null }, { "name": "nested_record", From a4b2a8c00ed068e72f827ea686d89fa4ad1a664c Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 15 Jan 2026 07:56:31 -0800 Subject: [PATCH 10/10] use iceberg 1.9.2 --- pom.xml | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 391a00487..e764aee26 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ 2.12 3.4.2 3.4 - 1.10.1 + 1.9.2 1.7.2 2.4.0 1.2.0 @@ -244,6 +244,11 @@ org.openjdk.jol jol-core + + + org.apache.parquet + * + @@ -263,6 +268,11 @@ org.apache.zookeeper zookeeper + + + org.apache.parquet + * + @@ -270,12 +280,26 @@ hudi-sync-common ${hudi.version} provided + + + + org.apache.parquet + * + + org.apache.hudi hudi-spark${spark.version.prefix}-bundle_${scala.binary.version} ${hudi.version} test + + + + org.apache.parquet + * + + org.apache.hudi @@ -284,11 +308,25 @@ tests test-jar test + + + + org.apache.parquet + * + + org.apache.hudi hudi-java-client ${hudi.version} + + + + org.apache.parquet + * + +