diff --git a/pom.xml b/pom.xml
index 855894ec2..e764aee26 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,7 +63,7 @@
2025-01-01T00:00:00Z
8
3.13.0
- 1.11.4
+ 1.12.0
2.22.0
5.11.4
1.11.4
@@ -87,7 +87,8 @@
2.12
3.4.2
3.4
- 1.4.2
+ 1.9.2
+ 1.7.2
2.4.0
1.2.0
2.18.2
@@ -243,6 +244,11 @@
org.openjdk.jol
jol-core
+
+
+ org.apache.parquet
+ *
+
@@ -262,6 +268,11 @@
org.apache.zookeeper
zookeeper
+
+
+ org.apache.parquet
+ *
+
@@ -269,12 +280,26 @@
hudi-sync-common
${hudi.version}
provided
+
+
+
+ org.apache.parquet
+ *
+
+
org.apache.hudi
hudi-spark${spark.version.prefix}-bundle_${scala.binary.version}
${hudi.version}
test
+
+
+
+ org.apache.parquet
+ *
+
+
org.apache.hudi
@@ -283,11 +308,25 @@
tests
test-jar
test
+
+
+
+ org.apache.parquet
+ *
+
+
org.apache.hudi
hudi-java-client
${hudi.version}
+
+
+
+ org.apache.parquet
+ *
+
+
@@ -315,7 +354,7 @@
org.apache.iceberg
iceberg-hive-runtime
- ${iceberg.version}
+ ${iceberg.hive.runtime.version}
@@ -748,6 +787,22 @@
true
false
120
+
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+ --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+ --add-opens=java.base/java.io=ALL-UNNAMED
+ --add-opens=java.base/java.net=ALL-UNNAMED
+ --add-opens=java.base/java.nio=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+ --add-opens=java.base/sun.security.action=ALL-UNNAMED
+ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+ --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+
@@ -767,7 +822,23 @@
false
6
false
- -Xmx1500m
+
+ -Xmx1500m
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+ --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+ --add-opens=java.base/java.io=ALL-UNNAMED
+ --add-opens=java.base/java.net=ALL-UNNAMED
+ --add-opens=java.base/java.nio=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+ --add-opens=java.base/sun.security.action=ALL-UNNAMED
+ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+ --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+
120
diff --git a/xtable-aws/src/main/resources/META-INF/LICENSE-bundled b/xtable-aws/src/main/resources/META-INF/LICENSE-bundled
index c73a8939c..70a583c08 100644
--- a/xtable-aws/src/main/resources/META-INF/LICENSE-bundled
+++ b/xtable-aws/src/main/resources/META-INF/LICENSE-bundled
@@ -226,8 +226,8 @@ org.apache.httpcomponents:fluent-hc:4.4.1
org.apache.hudi:hudi-common:0.14.0
org.apache.hudi:hudi-java-client:0.14.0
org.apache.hudi:hudi-timeline-service:0.14.0
-org.apache.iceberg:iceberg-api:1.4.2
-org.apache.iceberg:iceberg-core:1.4.2
+org.apache.iceberg:iceberg-api:1.10.1
+org.apache.iceberg:iceberg-core:1.10.1
org.apache.logging.log4j:log4j-1.2-api:2.22.0
org.apache.logging.log4j:log4j-api:2.22.0
org.javassist:javassist:3.25.0-GA
diff --git a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
index 82a094938..5a0b70cb9 100644
--- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
+++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
@@ -203,14 +203,18 @@ private static ColumnStat getColumnStatFromHudiStat(
if (field.getSchema().getDataType() == InternalType.DECIMAL) {
int scale =
(int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_SCALE);
- minValue =
- minValue instanceof ByteBuffer
- ? convertBytesToBigDecimal((ByteBuffer) minValue, scale)
- : ((BigDecimal) minValue).setScale(scale, RoundingMode.UNNECESSARY);
- maxValue =
- maxValue instanceof ByteBuffer
- ? convertBytesToBigDecimal((ByteBuffer) maxValue, scale)
- : ((BigDecimal) maxValue).setScale(scale, RoundingMode.UNNECESSARY);
+ if (minValue != null) {
+ minValue =
+ minValue instanceof ByteBuffer
+ ? convertBytesToBigDecimal((ByteBuffer) minValue, scale)
+ : ((BigDecimal) minValue).setScale(scale, RoundingMode.UNNECESSARY);
+ }
+ if (maxValue != null) {
+ maxValue =
+ maxValue instanceof ByteBuffer
+ ? convertBytesToBigDecimal((ByteBuffer) maxValue, scale)
+ : ((BigDecimal) maxValue).setScale(scale, RoundingMode.UNNECESSARY);
+ }
}
return getColumnStatFromValues(
minValue,
diff --git a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java
index e3da2e7d2..fccd5b22f 100644
--- a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java
+++ b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java
@@ -66,9 +66,6 @@ public InternalSchema toInternalSchema(StructType structType) {
return toInternalSchema(structType, null, false, null, null);
}
- String trimmedTypeName = "";
- InternalType type = null;
-
private InternalSchema toInternalSchema(
DataType dataType,
String parentPath,
@@ -76,6 +73,8 @@ private InternalSchema toInternalSchema(
String comment,
FieldMetadata originalMetadata) {
+ String trimmedTypeName = "";
+ InternalType type = null;
Map metadata = null;
List fields = null;
diff --git a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java
index b7b86578c..fe1414574 100644
--- a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java
+++ b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java
@@ -371,7 +371,7 @@ private DataFile writeAndGetDataFile(List records, StructLike partitionK
DataWriter dataWriter =
Parquet.writeData(file)
.schema(icebergTable.schema())
- .createWriterFunc(GenericParquetWriter::buildWriter)
+ .createWriterFunc(GenericParquetWriter::create)
.overwrite()
.withSpec(icebergTable.spec())
.withPartition(partitionKey)
diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java
index d1bf70bb6..10372e0b4 100644
--- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java
+++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java
@@ -432,7 +432,7 @@ private DataFile generateTestDataFile(int partition, Table table, String filePat
DataWriter dataWriter =
Parquet.writeData(table.io().newOutputFile(filePath))
.schema(csSchema)
- .createWriterFunc(GenericParquetWriter::buildWriter)
+ .createWriterFunc(GenericParquetWriter::create)
.overwrite()
.withSpec(table.spec())
.withPartition(partitionInfo)
diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java
index a8518c1f9..d53b72067 100644
--- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java
+++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java
@@ -333,11 +333,7 @@ private Object generateRandomValueForType(
case STRUCT:
return generateInsertRecord(timeLowerBound, timeUpperBound, fieldType.asStructType());
case UUID:
- UUID uuid = UUID.randomUUID();
- ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]);
- byteBuffer.putLong(uuid.getMostSignificantBits());
- byteBuffer.putLong(uuid.getLeastSignificantBits());
- return byteBuffer.array();
+ return UUID.randomUUID();
case LIST:
Types.ListType listType = (Types.ListType) fieldType;
int listSize = RANDOM.nextInt(5) + 1;
diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java
index b07fac4c5..ffdd5f996 100644
--- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java
+++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java
@@ -181,7 +181,7 @@ public void testUpdateListElement() {
verify(mockUpdateSchema).deleteColumn("array_field.element.element_string");
verify(mockUpdateSchema)
- .addColumn("array_field.element", "element_double", Types.DoubleType.get(), null);
+ .addColumn("array_field.element", "element_double", Types.DoubleType.get(), (String) null);
verify(mockUpdateSchema).commit();
}
@@ -194,7 +194,8 @@ public void testUpdateMapValue() {
verify(mockUpdateSchema).deleteColumn("map_field.value.value_string");
verify(mockUpdateSchema)
- .addRequiredColumn("map_field.value", "value_double", Types.DoubleType.get(), null);
+ .addRequiredColumn(
+ "map_field.value", "value_double", Types.DoubleType.get(), (String) null);
verify(mockUpdateSchema).commit();
}
diff --git a/xtable-core/src/test/resources/schemas/basic_schema.avsc b/xtable-core/src/test/resources/schemas/basic_schema.avsc
index 613094c12..37193be59 100644
--- a/xtable-core/src/test/resources/schemas/basic_schema.avsc
+++ b/xtable-core/src/test/resources/schemas/basic_schema.avsc
@@ -72,20 +72,23 @@
},
{
"name": "bytes_field",
- "type": "bytes",
- "default": ""
+ "type": ["null", "bytes"],
+ "default": null
},
{
"name": "decimal_field",
- "type": {
- "type": "fixed",
- "name": "decimal_field_testing",
- "size": 10,
- "logicalType": "decimal",
- "precision": 20,
- "scale": 2
- },
- "default": "\u0000"
+ "type": [
+ "null",
+ {
+ "type": "fixed",
+ "name": "decimal_field_testing",
+ "size": 10,
+ "logicalType": "decimal",
+ "precision": 20,
+ "scale": 2
+ }
+ ],
+ "default": null
},
{
"name": "nested_record",
diff --git a/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled b/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled
index 04fe28aa5..27c286a32 100644
--- a/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled
+++ b/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled
@@ -225,8 +225,8 @@ org.apache.httpcomponents:fluent-hc:4.4.1
org.apache.hudi:hudi-common:0.14.0
org.apache.hudi:hudi-java-client:0.14.0
org.apache.hudi:hudi-timeline-service:0.14.0
-org.apache.iceberg:iceberg-api:1.4.2
-org.apache.iceberg:iceberg-core:1.4.2
+org.apache.iceberg:iceberg-api:1.10.1
+org.apache.iceberg:iceberg-core:1.10.1
org.apache.logging.log4j:log4j-1.2-api:2.22.0
org.apache.logging.log4j:log4j-api:2.22.0
io.delta:delta-core_2.12:2.4.0
diff --git a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled
index 46089bf38..de1b2a89a 100644
--- a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled
+++ b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled
@@ -233,9 +233,9 @@ org.apache.hudi:hudi-client-common:0.14.0
org.apache.hudi:hudi-common:0.14.0
org.apache.hudi:hudi-timeline-service:0.14.0
org.apache.hudi:hudi-sync-common:0.14.0
-org.apache.iceberg:iceberg-api:1.4.2
-org.apache.iceberg:iceberg-core:1.4.2
-org.apache.iceberg:iceberg-parquet:1.4.2
+org.apache.iceberg:iceberg-api:1.10.1
+org.apache.iceberg:iceberg-core:1.10.1
+org.apache.iceberg:iceberg-parquet:1.10.1
org.apache.logging.log4j:log4j-api:2.22.0
org.apache.logging.log4j:log4j-1.2-api:2.22.0
org.roaringbitmap:RoaringBitmap:0.9.47