From 8efba9a55e3f529362c11296dfb3006b690b6be7 Mon Sep 17 00:00:00 2001 From: Shanu Gautam Date: Sat, 4 Jan 2025 19:26:02 +0530 Subject: [PATCH 1/4] testing schema failure fix --- cdi-core/build.gradle | 2 ++ .../src/main/java/com/linkedin/cdi/util/JsonElementTypes.java | 1 + 2 files changed, 3 insertions(+) diff --git a/cdi-core/build.gradle b/cdi-core/build.gradle index 407ea22..94d774b 100644 --- a/cdi-core/build.gradle +++ b/cdi-core/build.gradle @@ -57,6 +57,8 @@ dependencies { force = true } + implementation 'com.github.ben-manes.caffeine:caffeine:2.9.3' + runtime externalDependency.'gobblin-azkaban' runtime externalDependency.'gobblin-kafka-08' runtime externalDependency.'gobblin-kafka-common' diff --git a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java index d94030d..d0c5bd3 100644 --- a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java @@ -51,6 +51,7 @@ public enum JsonElementTypes { STRING("string"), TIME("time"), TIMESTAMP("timestamp"), + MAP("map", "map", false), UNION("union", "union", false), UNKNOWN("unknown"), NULL(true, "null", "null"), From 8f3997b0ef4a79d4514558f0b6cda4f3b86e40d9 Mon Sep 17 00:00:00 2001 From: Shanu Gautam Date: Wed, 4 Jun 2025 21:05:56 +0530 Subject: [PATCH 2/4] Nested Map JSON Schema Support --- .../com/linkedin/cdi/filter/JsonSchemaBasedFilter.java | 6 ++++++ .../com/linkedin/cdi/util/JsonIntermediateSchema.java | 8 ++++++++ version.properties | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cdi-core/src/main/java/com/linkedin/cdi/filter/JsonSchemaBasedFilter.java b/cdi-core/src/main/java/com/linkedin/cdi/filter/JsonSchemaBasedFilter.java index 85a9031..29da1cd 100644 --- a/cdi-core/src/main/java/com/linkedin/cdi/filter/JsonSchemaBasedFilter.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/filter/JsonSchemaBasedFilter.java @@ -45,6 +45,12 @@ private JsonElement filter(JsonIntermediateSchema.JisDataType dataType, JsonElem return filter(dataType.getChildRecord(), input.getAsJsonObject()); } else if (dataType.getType() == JsonElementTypes.ARRAY) { return filter(dataType.getItemType(), input.getAsJsonArray()); + } else if (dataType.getType() == JsonElementTypes.MAP) { + JsonObject output = new JsonObject(); + for (Map.Entry entry: input.getAsJsonObject().entrySet()) { + output.add(entry.getKey(), filter(dataType.getItemType().getChildRecord(), entry.getValue().getAsJsonObject())); + } + return output; } return null; } diff --git a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java index 09bc91a..28c2535 100644 --- a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java @@ -231,6 +231,14 @@ public void setSymbols(JsonArray symbols) { // an Enum has a list of symbols this.setSymbols(dataTypeDefinition.get(KEY_WORD_SYMBOLS).getAsJsonArray()); break; + case MAP: + JsonObject valuesObject = dataTypeDefinition.get(KEY_WORD_VALUES).getAsJsonObject(); + if(valuesObject.isJsonPrimitive()) { + this.setItemType(new JisDataType(valuesObject.getAsString())); + } else { + this.setItemType(new JisDataType(valuesObject.getAsJsonObject().get(KEY_WORD_DATA_TYPE).getAsJsonObject())); + } + break; case UNION: // a Union has 2 or more different types // TODO diff --git a/version.properties b/version.properties index 66f852b..fe121f4 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -version=0.2.117 \ No newline at end of file +version=0.2.118 \ No newline at end of file From 02f7f626692d230cb33d628db447796ff032c508 Mon Sep 17 00:00:00 2001 From: Shanu Gautam Date: Wed, 4 Jun 2025 21:27:49 +0530 Subject: [PATCH 3/4] Cleanup of test changes --- cdi-core/build.gradle | 2 -- .../src/main/java/com/linkedin/cdi/util/JsonElementTypes.java | 1 - 2 files changed, 3 deletions(-) diff --git a/cdi-core/build.gradle b/cdi-core/build.gradle index 94d774b..407ea22 100644 --- a/cdi-core/build.gradle +++ b/cdi-core/build.gradle @@ -57,8 +57,6 @@ dependencies { force = true } - implementation 'com.github.ben-manes.caffeine:caffeine:2.9.3' - runtime externalDependency.'gobblin-azkaban' runtime externalDependency.'gobblin-kafka-08' runtime externalDependency.'gobblin-kafka-common' diff --git a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java index 75564c1..5a8f427 100644 --- a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonElementTypes.java @@ -52,7 +52,6 @@ public enum JsonElementTypes { STRING("string"), TIME("time"), TIMESTAMP("timestamp"), - MAP("map", "map", false), UNION("union", "union", false), UNKNOWN("unknown"), NULL(true, "null", "null"), From a7d0f3fe7972da13710d4f402f4794b613491cb5 Mon Sep 17 00:00:00 2001 From: gautamshanu <96845064+gautamshanu@users.noreply.github.com> Date: Fri, 6 Jun 2025 00:41:19 +0530 Subject: [PATCH 4/4] Making primitive check before object conversion --- .../com/linkedin/cdi/util/JsonIntermediateSchema.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java index 28c2535..66c2a8e 100644 --- a/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java +++ b/cdi-core/src/main/java/com/linkedin/cdi/util/JsonIntermediateSchema.java @@ -232,11 +232,11 @@ public void setSymbols(JsonArray symbols) { this.setSymbols(dataTypeDefinition.get(KEY_WORD_SYMBOLS).getAsJsonArray()); break; case MAP: - JsonObject valuesObject = dataTypeDefinition.get(KEY_WORD_VALUES).getAsJsonObject(); - if(valuesObject.isJsonPrimitive()) { - this.setItemType(new JisDataType(valuesObject.getAsString())); + JsonElement valuesElement = dataTypeDefinition.get(KEY_WORD_VALUES); + if(valuesElement.isJsonPrimitive()) { + this.setItemType(new JisDataType(valuesElement.getAsString())); } else { - this.setItemType(new JisDataType(valuesObject.getAsJsonObject().get(KEY_WORD_DATA_TYPE).getAsJsonObject())); + this.setItemType(new JisDataType(valuesElement.getAsJsonObject().get(KEY_WORD_DATA_TYPE).getAsJsonObject())); } break; case UNION: