diff --git a/pom.xml b/pom.xml index 855894ec2..e764aee26 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 2025-01-01T00:00:00Z 8 3.13.0 - 1.11.4 + 1.12.0 2.22.0 5.11.4 1.11.4 @@ -87,7 +87,8 @@ 2.12 3.4.2 3.4 - 1.4.2 + 1.9.2 + 1.7.2 2.4.0 1.2.0 2.18.2 @@ -243,6 +244,11 @@ org.openjdk.jol jol-core + + + org.apache.parquet + * + @@ -262,6 +268,11 @@ org.apache.zookeeper zookeeper + + + org.apache.parquet + * + @@ -269,12 +280,26 @@ hudi-sync-common ${hudi.version} provided + + + + org.apache.parquet + * + + org.apache.hudi hudi-spark${spark.version.prefix}-bundle_${scala.binary.version} ${hudi.version} test + + + + org.apache.parquet + * + + org.apache.hudi @@ -283,11 +308,25 @@ tests test-jar test + + + + org.apache.parquet + * + + org.apache.hudi hudi-java-client ${hudi.version} + + + + org.apache.parquet + * + + @@ -315,7 +354,7 @@ org.apache.iceberg iceberg-hive-runtime - ${iceberg.version} + ${iceberg.hive.runtime.version} @@ -748,6 +787,22 @@ true false 120 + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED + @@ -767,7 +822,23 @@ false 6 false - -Xmx1500m + + -Xmx1500m + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED + 120 diff --git a/xtable-aws/src/main/resources/META-INF/LICENSE-bundled b/xtable-aws/src/main/resources/META-INF/LICENSE-bundled index c73a8939c..70a583c08 100644 --- a/xtable-aws/src/main/resources/META-INF/LICENSE-bundled +++ b/xtable-aws/src/main/resources/META-INF/LICENSE-bundled @@ -226,8 +226,8 @@ org.apache.httpcomponents:fluent-hc:4.4.1 org.apache.hudi:hudi-common:0.14.0 org.apache.hudi:hudi-java-client:0.14.0 org.apache.hudi:hudi-timeline-service:0.14.0 -org.apache.iceberg:iceberg-api:1.4.2 -org.apache.iceberg:iceberg-core:1.4.2 +org.apache.iceberg:iceberg-api:1.10.1 +org.apache.iceberg:iceberg-core:1.10.1 org.apache.logging.log4j:log4j-1.2-api:2.22.0 org.apache.logging.log4j:log4j-api:2.22.0 org.javassist:javassist:3.25.0-GA diff --git a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java index 82a094938..5a0b70cb9 100644 --- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java +++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java @@ -203,14 +203,18 @@ private static ColumnStat getColumnStatFromHudiStat( if (field.getSchema().getDataType() == InternalType.DECIMAL) { int scale = (int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_SCALE); - minValue = - minValue instanceof ByteBuffer - ? convertBytesToBigDecimal((ByteBuffer) minValue, scale) - : ((BigDecimal) minValue).setScale(scale, RoundingMode.UNNECESSARY); - maxValue = - maxValue instanceof ByteBuffer - ? convertBytesToBigDecimal((ByteBuffer) maxValue, scale) - : ((BigDecimal) maxValue).setScale(scale, RoundingMode.UNNECESSARY); + if (minValue != null) { + minValue = + minValue instanceof ByteBuffer + ? convertBytesToBigDecimal((ByteBuffer) minValue, scale) + : ((BigDecimal) minValue).setScale(scale, RoundingMode.UNNECESSARY); + } + if (maxValue != null) { + maxValue = + maxValue instanceof ByteBuffer + ? convertBytesToBigDecimal((ByteBuffer) maxValue, scale) + : ((BigDecimal) maxValue).setScale(scale, RoundingMode.UNNECESSARY); + } } return getColumnStatFromValues( minValue, diff --git a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java index e3da2e7d2..fccd5b22f 100644 --- a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java +++ b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java @@ -66,9 +66,6 @@ public InternalSchema toInternalSchema(StructType structType) { return toInternalSchema(structType, null, false, null, null); } - String trimmedTypeName = ""; - InternalType type = null; - private InternalSchema toInternalSchema( DataType dataType, String parentPath, @@ -76,6 +73,8 @@ private InternalSchema toInternalSchema( String comment, FieldMetadata originalMetadata) { + String trimmedTypeName = ""; + InternalType type = null; Map metadata = null; List fields = null; diff --git a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java index b7b86578c..fe1414574 100644 --- a/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java +++ b/xtable-core/src/test/java/org/apache/xtable/TestIcebergTable.java @@ -371,7 +371,7 @@ private DataFile writeAndGetDataFile(List records, StructLike partitionK DataWriter dataWriter = Parquet.writeData(file) .schema(icebergTable.schema()) - .createWriterFunc(GenericParquetWriter::buildWriter) + .createWriterFunc(GenericParquetWriter::create) .overwrite() .withSpec(icebergTable.spec()) .withPartition(partitionKey) diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java index d1bf70bb6..10372e0b4 100644 --- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java +++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergConversionSource.java @@ -432,7 +432,7 @@ private DataFile generateTestDataFile(int partition, Table table, String filePat DataWriter dataWriter = Parquet.writeData(table.io().newOutputFile(filePath)) .schema(csSchema) - .createWriterFunc(GenericParquetWriter::buildWriter) + .createWriterFunc(GenericParquetWriter::create) .overwrite() .withSpec(table.spec()) .withPartition(partitionInfo) diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java index a8518c1f9..d53b72067 100644 --- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java +++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergDataHelper.java @@ -333,11 +333,7 @@ private Object generateRandomValueForType( case STRUCT: return generateInsertRecord(timeLowerBound, timeUpperBound, fieldType.asStructType()); case UUID: - UUID uuid = UUID.randomUUID(); - ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[16]); - byteBuffer.putLong(uuid.getMostSignificantBits()); - byteBuffer.putLong(uuid.getLeastSignificantBits()); - return byteBuffer.array(); + return UUID.randomUUID(); case LIST: Types.ListType listType = (Types.ListType) fieldType; int listSize = RANDOM.nextInt(5) + 1; diff --git a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java index b07fac4c5..ffdd5f996 100644 --- a/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java +++ b/xtable-core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaSync.java @@ -181,7 +181,7 @@ public void testUpdateListElement() { verify(mockUpdateSchema).deleteColumn("array_field.element.element_string"); verify(mockUpdateSchema) - .addColumn("array_field.element", "element_double", Types.DoubleType.get(), null); + .addColumn("array_field.element", "element_double", Types.DoubleType.get(), (String) null); verify(mockUpdateSchema).commit(); } @@ -194,7 +194,8 @@ public void testUpdateMapValue() { verify(mockUpdateSchema).deleteColumn("map_field.value.value_string"); verify(mockUpdateSchema) - .addRequiredColumn("map_field.value", "value_double", Types.DoubleType.get(), null); + .addRequiredColumn( + "map_field.value", "value_double", Types.DoubleType.get(), (String) null); verify(mockUpdateSchema).commit(); } diff --git a/xtable-core/src/test/resources/schemas/basic_schema.avsc b/xtable-core/src/test/resources/schemas/basic_schema.avsc index 613094c12..37193be59 100644 --- a/xtable-core/src/test/resources/schemas/basic_schema.avsc +++ b/xtable-core/src/test/resources/schemas/basic_schema.avsc @@ -72,20 +72,23 @@ }, { "name": "bytes_field", - "type": "bytes", - "default": "" + "type": ["null", "bytes"], + "default": null }, { "name": "decimal_field", - "type": { - "type": "fixed", - "name": "decimal_field_testing", - "size": 10, - "logicalType": "decimal", - "precision": 20, - "scale": 2 - }, - "default": "\u0000" + "type": [ + "null", + { + "type": "fixed", + "name": "decimal_field_testing", + "size": 10, + "logicalType": "decimal", + "precision": 20, + "scale": 2 + } + ], + "default": null }, { "name": "nested_record", diff --git a/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled b/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled index 04fe28aa5..27c286a32 100644 --- a/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled +++ b/xtable-hive-metastore/src/main/resources/META-INF/LICENSE-bundled @@ -225,8 +225,8 @@ org.apache.httpcomponents:fluent-hc:4.4.1 org.apache.hudi:hudi-common:0.14.0 org.apache.hudi:hudi-java-client:0.14.0 org.apache.hudi:hudi-timeline-service:0.14.0 -org.apache.iceberg:iceberg-api:1.4.2 -org.apache.iceberg:iceberg-core:1.4.2 +org.apache.iceberg:iceberg-api:1.10.1 +org.apache.iceberg:iceberg-core:1.10.1 org.apache.logging.log4j:log4j-1.2-api:2.22.0 org.apache.logging.log4j:log4j-api:2.22.0 io.delta:delta-core_2.12:2.4.0 diff --git a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled index 46089bf38..de1b2a89a 100644 --- a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled +++ b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/resources/META-INF/LICENSE-bundled @@ -233,9 +233,9 @@ org.apache.hudi:hudi-client-common:0.14.0 org.apache.hudi:hudi-common:0.14.0 org.apache.hudi:hudi-timeline-service:0.14.0 org.apache.hudi:hudi-sync-common:0.14.0 -org.apache.iceberg:iceberg-api:1.4.2 -org.apache.iceberg:iceberg-core:1.4.2 -org.apache.iceberg:iceberg-parquet:1.4.2 +org.apache.iceberg:iceberg-api:1.10.1 +org.apache.iceberg:iceberg-core:1.10.1 +org.apache.iceberg:iceberg-parquet:1.10.1 org.apache.logging.log4j:log4j-api:2.22.0 org.apache.logging.log4j:log4j-1.2-api:2.22.0 org.roaringbitmap:RoaringBitmap:0.9.47