From d388c15d3afdacea3649e7ea9d3b2cfcbdeac1ee Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Mon, 16 Jun 2025 19:59:45 +0530 Subject: [PATCH 1/5] [ECO-5386] Defined LiveObjectSerializer interface 1. Added LiveObjectsHelper for initializing LiveObjects plugin and LiveObjectsSerializer impl. 2. Updated ProtocolMessage to handle msgpack serialization/deserialization for state field 3. Implemented LiveObjectsJsonSerializer for json serialization/deserialization of state field --- .../lib/objects/LiveObjectSerializer.java | 51 +++++++++++++++++++ .../ably/lib/objects/LiveObjectsHelper.java | 40 +++++++++++++++ .../objects/LiveObjectsJsonSerializer.java | 38 ++++++++++++++ .../io/ably/lib/realtime/AblyRealtime.java | 20 +------- .../io/ably/lib/types/ProtocolMessage.java | 32 ++++++++++++ live-objects/build.gradle.kts | 1 + .../io/ably/lib/objects/Serialization.kt | 21 ++++++++ 7 files changed, 185 insertions(+), 18 deletions(-) create mode 100644 lib/src/main/java/io/ably/lib/objects/LiveObjectSerializer.java create mode 100644 lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java create mode 100644 lib/src/main/java/io/ably/lib/objects/LiveObjectsJsonSerializer.java diff --git a/lib/src/main/java/io/ably/lib/objects/LiveObjectSerializer.java b/lib/src/main/java/io/ably/lib/objects/LiveObjectSerializer.java new file mode 100644 index 000000000..dcf0ce5cb --- /dev/null +++ b/lib/src/main/java/io/ably/lib/objects/LiveObjectSerializer.java @@ -0,0 +1,51 @@ +package io.ably.lib.objects; + +import com.google.gson.JsonArray; +import org.jetbrains.annotations.NotNull; +import org.msgpack.core.MessagePacker; +import org.msgpack.core.MessageUnpacker; + +import java.io.IOException; + +/** + * Serializer interface for converting between LiveObject arrays and their + * MessagePack or JSON representations. + */ +public interface LiveObjectSerializer { + /** + * Reads a MessagePack array from the given unpacker and deserializes it into an Object array. + * + * @param unpacker the MessageUnpacker to read from + * @return the deserialized Object array + * @throws IOException if an I/O error occurs during unpacking + */ + @NotNull + Object[] readMsgpackArray(@NotNull MessageUnpacker unpacker) throws IOException; + + /** + * Serializes the given Object array as a MessagePack array using the provided packer. + * + * @param objects the Object array to serialize + * @param packer the MessagePacker to write to + * @throws IOException if an I/O error occurs during packing + */ + void writeMsgpackArray(@NotNull Object[] objects, @NotNull MessagePacker packer) throws IOException; + + /** + * Reads a JSON array from the given {@link JsonArray} and deserializes it into an Object array. + * + * @param json the {@link JsonArray} representing the array to deserialize + * @return the deserialized Object array + */ + @NotNull + Object[] readFromJsonArray(@NotNull JsonArray json); + + /** + * Serializes the given Object array as a JSON array. + * + * @param objects the Object array to serialize + * @return the resulting JsonArray + */ + @NotNull + JsonArray asJsonArray(@NotNull Object[] objects); +} diff --git a/lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java b/lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java new file mode 100644 index 000000000..241d5d0d2 --- /dev/null +++ b/lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java @@ -0,0 +1,40 @@ +package io.ably.lib.objects; + +import io.ably.lib.realtime.AblyRealtime; +import io.ably.lib.util.Log; + +import java.lang.reflect.InvocationTargetException; + +public class LiveObjectsHelper { + + private static final String TAG = LiveObjectsHelper.class.getName(); + private static LiveObjectSerializer liveObjectSerializer; + + public static LiveObjectsPlugin tryInitializeLiveObjectsPlugin(AblyRealtime ablyRealtime) { + try { + Class liveObjectsImplementation = Class.forName("io.ably.lib.objects.DefaultLiveObjectsPlugin"); + LiveObjectsAdapter adapter = new Adapter(ablyRealtime); + return (LiveObjectsPlugin) liveObjectsImplementation + .getDeclaredConstructor(LiveObjectsAdapter.class) + .newInstance(adapter); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | + InvocationTargetException e) { + Log.i(TAG, "LiveObjects plugin not found in classpath. LiveObjects functionality will not be available.", e); + return null; + } + } + + public static LiveObjectSerializer getLiveObjectSerializer() { + if (liveObjectSerializer == null) { + try { + Class serializerClass = Class.forName("io.ably.lib.objects.DefaultLiveObjectSerializer"); + liveObjectSerializer = (LiveObjectSerializer) serializerClass.getDeclaredConstructor().newInstance(); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | + InvocationTargetException e) { + Log.e(TAG, "Failed to init LiveObjectSerializer, LiveObjects plugin not included in the classpath", e); + return null; + } + } + return liveObjectSerializer; + } +} diff --git a/lib/src/main/java/io/ably/lib/objects/LiveObjectsJsonSerializer.java b/lib/src/main/java/io/ably/lib/objects/LiveObjectsJsonSerializer.java new file mode 100644 index 000000000..f6a843474 --- /dev/null +++ b/lib/src/main/java/io/ably/lib/objects/LiveObjectsJsonSerializer.java @@ -0,0 +1,38 @@ +package io.ably.lib.objects; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import io.ably.lib.util.Log; + +import java.lang.reflect.Type; + +public class LiveObjectsJsonSerializer implements JsonSerializer, JsonDeserializer { + private static final String TAG = LiveObjectsJsonSerializer.class.getName(); + private final LiveObjectSerializer serializer = LiveObjectsHelper.getLiveObjectSerializer(); + + @Override + public Object[] deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + if (serializer == null) { + Log.w(TAG, "Skipping 'state' field json deserialization because LiveObjectsSerializer not found."); + return null; + } + if (!json.isJsonArray()) { + throw new JsonParseException("Expected a JSON array for 'state' field, but got: " + json); + } + return serializer.readFromJsonArray(json.getAsJsonArray()); + } + + @Override + public JsonElement serialize(Object[] src, Type typeOfSrc, JsonSerializationContext context) { + if (serializer == null) { + Log.w(TAG, "Skipping 'state' field json serialization because LiveObjectsSerializer not found."); + return JsonNull.INSTANCE; + } + return serializer.asJsonArray(src); + } +} diff --git a/lib/src/main/java/io/ably/lib/realtime/AblyRealtime.java b/lib/src/main/java/io/ably/lib/realtime/AblyRealtime.java index a933a7f62..8c0d9ee03 100644 --- a/lib/src/main/java/io/ably/lib/realtime/AblyRealtime.java +++ b/lib/src/main/java/io/ably/lib/realtime/AblyRealtime.java @@ -1,13 +1,11 @@ package io.ably.lib.realtime; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import io.ably.lib.objects.Adapter; -import io.ably.lib.objects.LiveObjectsAdapter; +import io.ably.lib.objects.LiveObjectsHelper; import io.ably.lib.objects.LiveObjectsPlugin; import io.ably.lib.rest.AblyRest; import io.ably.lib.rest.Auth; @@ -74,7 +72,7 @@ public AblyRealtime(ClientOptions options) throws AblyException { final InternalChannels channels = new InternalChannels(); this.channels = channels; - liveObjectsPlugin = tryInitializeLiveObjectsPlugin(); + liveObjectsPlugin = LiveObjectsHelper.tryInitializeLiveObjectsPlugin(this); connection = new Connection(this, channels, platformAgentProvider, liveObjectsPlugin); @@ -185,20 +183,6 @@ public interface Channels extends ReadOnlyMap { void release(String channelName); } - private LiveObjectsPlugin tryInitializeLiveObjectsPlugin() { - try { - Class liveObjectsImplementation = Class.forName("io.ably.lib.objects.DefaultLiveObjectsPlugin"); - LiveObjectsAdapter adapter = new Adapter(this); - return (LiveObjectsPlugin) liveObjectsImplementation - .getDeclaredConstructor(LiveObjectsAdapter.class) - .newInstance(adapter); - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | - InvocationTargetException e) { - Log.i(TAG, "LiveObjects plugin not found in classpath. LiveObjects functionality will not be available.", e); - return null; - } - } - private class InternalChannels extends InternalMap implements Channels, ConnectionManager.Channels { /** * Get the named channel; if it does not already exist, diff --git a/lib/src/main/java/io/ably/lib/types/ProtocolMessage.java b/lib/src/main/java/io/ably/lib/types/ProtocolMessage.java index 73db3bf23..eea91b9ce 100644 --- a/lib/src/main/java/io/ably/lib/types/ProtocolMessage.java +++ b/lib/src/main/java/io/ably/lib/types/ProtocolMessage.java @@ -4,6 +4,11 @@ import java.lang.reflect.Type; import java.util.Map; +import com.google.gson.annotations.JsonAdapter; +import io.ably.lib.objects.LiveObjectSerializer; +import io.ably.lib.objects.LiveObjectsHelper; +import io.ably.lib.objects.LiveObjectsJsonSerializer; +import org.jetbrains.annotations.Nullable; import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePacker; import org.msgpack.core.MessageUnpacker; @@ -123,6 +128,14 @@ public ProtocolMessage(Action action, String channel) { public AuthDetails auth; public Map params; public Annotation[] annotations; + /** + * This will be null if we skipped decoding this property due to user not requesting Objects functionality + * JsonAdapter annotation supports java version (1.8) mentioned in build.gradle + * This is targeted and specific to the state field, so won't affect other fields + */ + @Nullable + @JsonAdapter(LiveObjectsJsonSerializer.class) + public Object[] state; public boolean hasFlag(final Flag flag) { return (flags & flag.getMask()) == flag.getMask(); @@ -147,6 +160,7 @@ void writeMsgpack(MessagePacker packer) throws IOException { if(params != null) ++fieldCount; if(channelSerial != null) ++fieldCount; if(annotations != null) ++fieldCount; + if(state != null) ++fieldCount; packer.packMapHeader(fieldCount); packer.packString("action"); packer.packInt(action.getValue()); @@ -186,6 +200,15 @@ void writeMsgpack(MessagePacker packer) throws IOException { packer.packString("annotations"); AnnotationSerializer.writeMsgpackArray(annotations, packer); } + if(state != null) { + LiveObjectSerializer liveObjectsSerializer = LiveObjectsHelper.getLiveObjectSerializer(); + if (liveObjectsSerializer != null) { + packer.packString("state"); + liveObjectsSerializer.writeMsgpackArray(state, packer); + } else { + Log.w(TAG, "Skipping 'state' field msgpack serialization because LiveObjectsSerializer not found"); + } + } } ProtocolMessage readMsgpack(MessageUnpacker unpacker) throws IOException { @@ -248,6 +271,15 @@ ProtocolMessage readMsgpack(MessageUnpacker unpacker) throws IOException { case "annotations": annotations = AnnotationSerializer.readMsgpackArray(unpacker); break; + case "state": + LiveObjectSerializer liveObjectsSerializer = LiveObjectsHelper.getLiveObjectSerializer(); + if (liveObjectsSerializer != null) { + state = liveObjectsSerializer.readMsgpackArray(unpacker); + } else { + Log.w(TAG, "Skipping 'state' field msgpack deserialization because LiveObjectsSerializer not found"); + unpacker.skipValue(); + } + break; default: Log.v(TAG, "Unexpected field: " + fieldName); unpacker.skipValue(); diff --git a/live-objects/build.gradle.kts b/live-objects/build.gradle.kts index 2adb88fff..ce6642496 100644 --- a/live-objects/build.gradle.kts +++ b/live-objects/build.gradle.kts @@ -11,6 +11,7 @@ repositories { dependencies { implementation(project(":java")) + implementation(libs.bundles.common) implementation(libs.coroutine.core) testImplementation(kotlin("test")) diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt index e2279d843..6e67d5fef 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt @@ -2,9 +2,30 @@ package io.ably.lib.objects import com.google.gson.Gson import com.google.gson.GsonBuilder +import com.google.gson.JsonArray +import org.msgpack.core.MessagePacker +import org.msgpack.core.MessageUnpacker internal val gson: Gson = createGsonSerializer() private fun createGsonSerializer(): Gson { return GsonBuilder().create() // Do not call serializeNulls() to omit null values } + +internal class DefaultLiveObjectSerializer : LiveObjectSerializer { + override fun readMsgpackArray(unpacker: MessageUnpacker): Array { + TODO("Not yet implemented") + } + + override fun writeMsgpackArray(objects: Array?, packer: MessagePacker) { + TODO("Not yet implemented") + } + + override fun readFromJsonArray(json: JsonArray): Array { + TODO("Not yet implemented") + } + + override fun asJsonArray(objects: Array?): JsonArray { + TODO("Not yet implemented") + } +} From 7a4873e6ce945a6a0cf9777bca7ff1feca86185f Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Tue, 17 Jun 2025 18:45:01 +0530 Subject: [PATCH 2/5] [ECO-5386] Implement JSON and MessagePack serialization for ObjectMessage 1. Added jackson-dataformat-msgpack 0.8.11 dependency for automatic msgpack handling 2. Implemented JSON serialization via ObjectMessage.toJsonObject() and JsonObject.toObjectMessage() extensions 3. Implemented MessagePack serialization via ObjectMessage.writeTo() and MessageUnpacker.readObjectMessage() extensions 4. Added @SerializedName("object") annotation for proper field naming consistency --- gradle/libs.versions.toml | 2 + live-objects/build.gradle.kts | 1 + .../io/ably/lib/objects/ObjectMessage.kt | 5 ++ .../io/ably/lib/objects/Serialization.kt | 77 ++++++++++++++++--- 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 62b9b1f02..b8b1dfd6a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -25,6 +25,7 @@ mockk = "1.14.2" turbine = "1.2.0" ktor = "3.1.3" jetbrains-annoations = "26.0.2" +jackson-msgpack = "0.8.11" # Compatible with msgpack-core 0.8.11 [libraries] gson = { group = "com.google.code.gson", name = "gson", version.ref = "gson" } @@ -54,6 +55,7 @@ turbine = { group = "app.cash.turbine", name = "turbine", version.ref = "turbine ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } jetbrains = { group = "org.jetbrains", name = "annotations", version.ref = "jetbrains-annoations" } +jackson-msgpack = { group = "org.msgpack", name = "jackson-dataformat-msgpack", version.ref = "jackson-msgpack" } [bundles] common = ["msgpack", "vcdiff-core"] diff --git a/live-objects/build.gradle.kts b/live-objects/build.gradle.kts index ce6642496..7cbd15fd2 100644 --- a/live-objects/build.gradle.kts +++ b/live-objects/build.gradle.kts @@ -13,6 +13,7 @@ dependencies { implementation(project(":java")) implementation(libs.bundles.common) implementation(libs.coroutine.core) + implementation(libs.jackson.msgpack) testImplementation(kotlin("test")) testImplementation(libs.bundles.kotlin.tests) diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt index be168f993..625af8204 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt @@ -3,6 +3,9 @@ package io.ably.lib.objects import com.google.gson.JsonArray import com.google.gson.JsonObject +import com.fasterxml.jackson.annotation.JsonProperty +import com.google.gson.annotations.SerializedName + /** * An enum class representing the different actions that can be performed on an object. * Spec: OOP2 @@ -328,6 +331,8 @@ internal data class ObjectMessage( * the `ProtocolMessage` encapsulating it is `OBJECT_SYNC`. * Spec: OM2g */ + @SerializedName("object") + @JsonProperty("object") val objectState: ObjectState? = null, /** diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt index 6e67d5fef..b857d7b39 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt @@ -1,31 +1,88 @@ +@file:Suppress("UNCHECKED_CAST") + package io.ably.lib.objects -import com.google.gson.Gson -import com.google.gson.GsonBuilder -import com.google.gson.JsonArray +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.gson.* +import org.msgpack.core.MessagePack import org.msgpack.core.MessagePacker import org.msgpack.core.MessageUnpacker +import org.msgpack.jackson.dataformat.MessagePackFactory + +// Gson instance for JSON serialization/deserialization +internal val gson: Gson = GsonBuilder().create() -internal val gson: Gson = createGsonSerializer() +// Jackson ObjectMapper for MessagePack serialization (respects @JsonProperty annotations) +// Caches type metadata and serializers for ObjectMessage class after first use, so next time it's super fast 🚀 +private val msgpackMapper = ObjectMapper(MessagePackFactory()) + +internal fun ObjectMessage.toJsonObject(): JsonObject { + return gson.toJsonTree(this).asJsonObject +} -private fun createGsonSerializer(): Gson { - return GsonBuilder().create() // Do not call serializeNulls() to omit null values +internal fun JsonObject.toObjectMessage(): ObjectMessage { + return gson.fromJson(this, ObjectMessage::class.java) } +internal fun ObjectMessage.writeTo(packer: MessagePacker) { + // Jackson automatically creates the correct msgpack map structure + val msgpackBytes = msgpackMapper.writeValueAsBytes(this) + + // Parse the msgpack bytes to get the structured value + val tempUnpacker = MessagePack.newDefaultUnpacker(msgpackBytes) + val msgpackValue = tempUnpacker.unpackValue() + tempUnpacker.close() + + // Pack the structured value using the provided packer + packer.packValue(msgpackValue) +} + +internal fun MessageUnpacker.readObjectMessage(): ObjectMessage { + // Read the msgpack value from the unpacker + val msgpackValue = this.unpackValue() + + // Convert the msgpack value back to bytes + val tempPacker = MessagePack.newDefaultBufferPacker() + tempPacker.packValue(msgpackValue) + val msgpackBytes = tempPacker.toByteArray() + tempPacker.close() + + // Let Jackson deserialize the msgpack bytes back to ObjectMessage + return msgpackMapper.readValue(msgpackBytes, ObjectMessage::class.java) +} + +/** + * Default implementation of {@link LiveObjectSerializer} that handles serialization/deserialization + * of ObjectMessage arrays for both JSON and MessagePack formats using Jackson and Gson. + * Dynamically loaded by LiveObjectsHelper#getLiveObjectSerializer() to avoid hard dependencies. + */ +@Suppress("unused") // Used via reflection in LiveObjectsHelper internal class DefaultLiveObjectSerializer : LiveObjectSerializer { + override fun readMsgpackArray(unpacker: MessageUnpacker): Array { - TODO("Not yet implemented") + val objectMessagesCount = unpacker.unpackArrayHeader() + return Array(objectMessagesCount) { unpacker.readObjectMessage() } } override fun writeMsgpackArray(objects: Array?, packer: MessagePacker) { - TODO("Not yet implemented") + val objectMessages: Array = objects as Array + packer.packArrayHeader(objectMessages.size) + objectMessages.forEach { it.writeTo(packer) } } override fun readFromJsonArray(json: JsonArray): Array { - TODO("Not yet implemented") + return json.map { element -> + if (element.isJsonObject) element.asJsonObject.toObjectMessage() + else throw JsonParseException("Expected JsonObject, but found: $element") + }.toTypedArray() } override fun asJsonArray(objects: Array?): JsonArray { - TODO("Not yet implemented") + val objectMessages: Array = objects as Array + val jsonArray = JsonArray() + for (objectMessage in objectMessages) { + jsonArray.add(objectMessage.toJsonObject()) + } + return jsonArray } } From b846865e401330673478395a78d137a6dfe6a7d8 Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Wed, 18 Jun 2025 17:57:23 +0530 Subject: [PATCH 3/5] [ECO-5386] Implemented custom serializers for ObjectData type 1. Added JSON serializer/deserializer using ObjectDataJsonSerializer 2, Added msgpack serializer/deserializer using ObjectDataMsgpackSerializer and ObjectDataMsgpackDeserializer --- .../io/ably/lib/objects/ObjectMessage.kt | 27 ++--- .../io/ably/lib/objects/Serialization.kt | 100 ++++++++++++++++++ .../lib/objects/unit/ObjectMessageSizeTest.kt | 4 +- 3 files changed, 112 insertions(+), 19 deletions(-) diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt index 625af8204..83ec6b82a 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt @@ -4,6 +4,9 @@ import com.google.gson.JsonArray import com.google.gson.JsonObject import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import com.fasterxml.jackson.databind.annotation.JsonSerialize +import com.google.gson.annotations.JsonAdapter import com.google.gson.annotations.SerializedName /** @@ -31,6 +34,9 @@ internal enum class MapSemantics(val code: Int) { * An ObjectData represents a value in an object on a channel. * Spec: OD1 */ +@JsonAdapter(ObjectDataJsonSerializer::class) +@JsonSerialize(using = ObjectDataMsgpackSerializer::class) +@JsonDeserialize(using = ObjectDataMsgpackDeserializer::class) internal data class ObjectData( /** * A reference to another object, used to support composable object structures. @@ -38,12 +44,6 @@ internal data class ObjectData( */ val objectId: String? = null, - /** - * Can be set by the client to indicate that value in `string` or `bytes` field have an encoding. - * Spec: OD2b - */ - val encoding: String? = null, - /** * String, number, boolean or binary - a concrete value of the object * Spec: OD2c @@ -214,18 +214,13 @@ internal data class ObjectOperation( val nonce: String? = null, /** - * The initial value bytes for the object. These bytes should be used along with the nonce - * and timestamp to create the object ID. Frontdoor will use this to verify the object ID. - * After verification the bytes will be decoded into the Map or Counter objects and - * the initialValue, nonce, and initialValueEncoding will be removed. + * The initial value for the object, encoded as a JSON string. + * This value should be used along with the nonce and timestamp to create the object ID. + * Frontdoor will use this to verify the object ID. After verification, the value will be + * decoded into the Map or Counter objects and the initialValue, nonce, and initialValueEncoding will be removed. * Spec: OOP3h */ - val initialValue: Binary? = null, - - /** The initial value encoding defines how the initialValue should be interpreted. - * Spec: OOP3i - */ - val initialValueEncoding: ProtocolMessageFormat? = null + val initialValue: String? = null, ) /** diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt index b857d7b39..27c6b5655 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt @@ -2,12 +2,17 @@ package io.ably.lib.objects +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializerProvider import com.google.gson.* import org.msgpack.core.MessagePack import org.msgpack.core.MessagePacker import org.msgpack.core.MessageUnpacker import org.msgpack.jackson.dataformat.MessagePackFactory +import java.lang.reflect.Type +import java.util.* // Gson instance for JSON serialization/deserialization internal val gson: Gson = GsonBuilder().create() @@ -86,3 +91,98 @@ internal class DefaultLiveObjectSerializer : LiveObjectSerializer { return jsonArray } } + +internal class ObjectDataJsonSerializer : JsonSerializer, JsonDeserializer { + override fun serialize(src: ObjectData?, typeOfSrc: Type?, context: JsonSerializationContext?): JsonElement { + val obj = JsonObject() + src?.objectId?.let { obj.addProperty("objectId", it) } + + src?.value?.let { value -> + when (val v = value.value) { + is Boolean -> obj.addProperty("boolean", v) + is String -> obj.addProperty("string", v) + is Number -> obj.addProperty("number", v) + is Binary -> obj.addProperty("bytes", Base64.getEncoder().encodeToString(v.data)) + // Spec: OD4c5 + is JsonObject, is JsonArray -> { + obj.addProperty("string", v.toString()) + obj.addProperty("encoding", "json") + } + } + } + return obj + } + + override fun deserialize(json: JsonElement?, typeOfT: Type?, context: JsonDeserializationContext?): ObjectData { + val obj = if (json?.isJsonObject == true) json.asJsonObject else throw JsonParseException("Expected JsonObject") + val objectId = if (obj.has("objectId")) obj.get("objectId").asString else null + val encoding = if (obj.has("encoding")) obj.get("encoding").asString else null + val value = when { + obj.has("boolean") -> ObjectValue(obj.get("boolean").asBoolean) + // Spec: OD5b3 + obj.has("string") && encoding == "json" -> { + val jsonStr = obj.get("string").asString + val parsed = JsonParser.parseString(jsonStr) + ObjectValue( + when { + parsed.isJsonObject -> parsed.asJsonObject + parsed.isJsonArray -> parsed.asJsonArray + else -> throw JsonParseException("Invalid JSON string for encoding=json") + } + ) + } + obj.has("string") -> ObjectValue(obj.get("string").asString) + obj.has("number") -> ObjectValue(obj.get("number").asNumber) + obj.has("bytes") -> ObjectValue(Binary(Base64.getDecoder().decode(obj.get("bytes").asString))) + else -> throw JsonParseException("ObjectData must have one of the fields: boolean, string, number, or bytes") + } + return ObjectData(objectId, value) + } +} + +internal class ObjectDataMsgpackSerializer : com.fasterxml.jackson.databind.JsonSerializer() { + override fun serialize(value: ObjectData?, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeStartObject() + value?.objectId?.let { gen.writeStringField("objectId", it) } + value?.value?.let { v -> + when (val data = v.value) { + is Boolean -> gen.writeBooleanField("boolean", data) + is String -> gen.writeStringField("string", data) + is Number -> gen.writeNumberField("number", data.toDouble()) + is Binary -> gen.writeBinaryField("bytes", data.data) + is JsonObject, is JsonArray -> { + gen.writeStringField("string", data.toString()) + gen.writeStringField("encoding", "json") + } + } + } + gen.writeEndObject() + } +} + +internal class ObjectDataMsgpackDeserializer : com.fasterxml.jackson.databind.JsonDeserializer() { + override fun deserialize(p: com.fasterxml.jackson.core.JsonParser, ctxt: DeserializationContext): ObjectData { + val node = p.codec.readTree(p) + val objectId = node.get("objectId")?.asText() + val encoding = node.get("encoding")?.asText() + val value = when { + node.has("boolean") -> ObjectValue(node.get("boolean").asBoolean()) + node.has("string") && encoding == "json" -> { + val jsonStr = node.get("string").asText() + val parsed = JsonParser.parseString(jsonStr) + ObjectValue( + when { + parsed.isJsonObject -> parsed.asJsonObject + parsed.isJsonArray -> parsed.asJsonArray + else -> throw IllegalArgumentException("Invalid JSON string for encoding=json") + } + ) + } + node.has("string") -> ObjectValue(node.get("string").asText()) + node.has("number") -> ObjectValue(node.get("number").numberValue()) + node.has("bytes") -> ObjectValue(Binary(node.get("bytes").binaryValue())) + else -> throw IllegalArgumentException("ObjectData must have one of the fields: boolean, string, number, or bytes") + } + return ObjectData(objectId, value) + } +} diff --git a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt index f4d368d89..9de77219f 100644 --- a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt +++ b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt @@ -45,7 +45,6 @@ class ObjectMessageSizeTest { key = "mapKey", // Size: 6 bytes (UTF-8 byte length) data = ObjectData( objectId = "ref_obj", // Not counted in data size - encoding = "utf-8", // Not counted in data size value = ObjectValue("sample") // Size: 6 bytes (UTF-8 byte length) ) // Total ObjectData size: 6 bytes ), // Total ObjectMapOp size: 6 + 6 = 12 bytes @@ -80,8 +79,7 @@ class ObjectMessageSizeTest { ), // Total ObjectCounter size: 8 bytes nonce = "nonce123", // Not counted in operation size - initialValue = Binary("initial".toByteArray()), // Not counted in operation size - initialValueEncoding = ProtocolMessageFormat.Json // Not counted in operation size + initialValue = "some-value", // Not counted in operation size ), // Total ObjectOperation size: 12 + 8 + 26 + 8 = 54 bytes objectState = ObjectState( From 7e4f5d689ed4990644244097081e9ab819e790fd Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Fri, 20 Jun 2025 16:41:00 +0530 Subject: [PATCH 4/5] [ECO-5386] Implemented unit tests for ObjectMesssage json/msgpack serialization 1. Created ObjectMessageFixtures to represent dummy data in various formats 2. Added jackson-param dependency to fix jackson deserialization issue 3. Marked javaParameters as true on compilerOptions --- gradle/libs.versions.toml | 2 + live-objects/build.gradle.kts | 10 + .../io/ably/lib/objects/ObjectMessage.kt | 19 +- .../io/ably/lib/objects/Serialization.kt | 68 ++++--- .../unit/ObjectMessageSerializationTest.kt | 78 ++++++++ .../lib/objects/unit/ObjectMessageSizeTest.kt | 4 +- .../unit/fixtures/ObjectMessageFixture.kt | 174 ++++++++++++++++++ 7 files changed, 323 insertions(+), 32 deletions(-) create mode 100644 live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt create mode 100644 live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b8b1dfd6a..84517b90c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -26,6 +26,7 @@ turbine = "1.2.0" ktor = "3.1.3" jetbrains-annoations = "26.0.2" jackson-msgpack = "0.8.11" # Compatible with msgpack-core 0.8.11 +jackson-param = "2.18.0" # Compatible with jackson-msgpack [libraries] gson = { group = "com.google.code.gson", name = "gson", version.ref = "gson" } @@ -56,6 +57,7 @@ ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } jetbrains = { group = "org.jetbrains", name = "annotations", version.ref = "jetbrains-annoations" } jackson-msgpack = { group = "org.msgpack", name = "jackson-dataformat-msgpack", version.ref = "jackson-msgpack" } +jackson-parameter-names = { group = "com.fasterxml.jackson.module", name = "jackson-module-parameter-names", version.ref = "jackson-param" } [bundles] common = ["msgpack", "vcdiff-core"] diff --git a/live-objects/build.gradle.kts b/live-objects/build.gradle.kts index 7cbd15fd2..3c951ce64 100644 --- a/live-objects/build.gradle.kts +++ b/live-objects/build.gradle.kts @@ -14,6 +14,8 @@ dependencies { implementation(libs.bundles.common) implementation(libs.coroutine.core) implementation(libs.jackson.msgpack) + implementation(libs.jackson.parameter.names) // Add this + testImplementation(kotlin("test")) testImplementation(libs.bundles.kotlin.tests) @@ -45,4 +47,12 @@ tasks.register("runLiveObjectIntegrationTests") { kotlin { explicitApi() + + /** + * Enables Jackson to map JSON property names to constructor parameters without use of @JsonProperty. + * Adds metadata params to bytecode class. Approach is completely binary-compatible with consumers of the library. + */ + compilerOptions { + javaParameters = true + } } diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt index 83ec6b82a..1a694a132 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt @@ -214,13 +214,22 @@ internal data class ObjectOperation( val nonce: String? = null, /** - * The initial value for the object, encoded as a JSON string. - * This value should be used along with the nonce and timestamp to create the object ID. - * Frontdoor will use this to verify the object ID. After verification, the value will be - * decoded into the Map or Counter objects and the initialValue, nonce, and initialValueEncoding will be removed. + * The initial value bytes for the object. These bytes should be used along with the nonce + * and timestamp to create the object ID. Frontdoor will use this to verify the object ID. + * After verification the bytes will be decoded into the Map or Counter objects and + * the initialValue, nonce, and initialValueEncoding will be removed. * Spec: OOP3h */ - val initialValue: String? = null, + @JsonAdapter(InitialValueJsonSerializer::class) + @JsonSerialize(using = InitialValueMsgpackSerializer::class) + @JsonDeserialize(using = InitialValueMsgpackDeserializer::class) + val initialValue: Binary? = null, + + /** The initial value encoding defines how the initialValue should be interpreted. + * Spec: OOP3i + */ + @Deprecated("Will be removed in the future, initialValue will be json string") + val initialValueEncoding: ProtocolMessageFormat? = null ) /** diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt index 27c6b5655..319cb33b8 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt @@ -2,15 +2,19 @@ package io.ably.lib.objects +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule import com.google.gson.* import org.msgpack.core.MessagePack import org.msgpack.core.MessagePacker import org.msgpack.core.MessageUnpacker import org.msgpack.jackson.dataformat.MessagePackFactory +import org.msgpack.value.ImmutableMapValue import java.lang.reflect.Type import java.util.* @@ -19,7 +23,11 @@ internal val gson: Gson = GsonBuilder().create() // Jackson ObjectMapper for MessagePack serialization (respects @JsonProperty annotations) // Caches type metadata and serializers for ObjectMessage class after first use, so next time it's super fast 🚀 -private val msgpackMapper = ObjectMapper(MessagePackFactory()) +// https://github.com/FasterXML/jackson-modules-java8/tree/3.x/parameter-names +internal val msgpackMapper = ObjectMapper(MessagePackFactory()).apply { + registerModule(ParameterNamesModule(JsonCreator.Mode.PROPERTIES)) + setSerializationInclusion(JsonInclude.Include.NON_NULL) +} internal fun ObjectMessage.toJsonObject(): JsonObject { return gson.toJsonTree(this).asJsonObject @@ -30,29 +38,15 @@ internal fun JsonObject.toObjectMessage(): ObjectMessage { } internal fun ObjectMessage.writeTo(packer: MessagePacker) { - // Jackson automatically creates the correct msgpack map structure - val msgpackBytes = msgpackMapper.writeValueAsBytes(this) - - // Parse the msgpack bytes to get the structured value - val tempUnpacker = MessagePack.newDefaultUnpacker(msgpackBytes) - val msgpackValue = tempUnpacker.unpackValue() - tempUnpacker.close() - - // Pack the structured value using the provided packer - packer.packValue(msgpackValue) + val msgpackBytes = msgpackMapper.writeValueAsBytes(this) // returns correct msgpack map structure + packer.writePayload(msgpackBytes) } -internal fun MessageUnpacker.readObjectMessage(): ObjectMessage { - // Read the msgpack value from the unpacker - val msgpackValue = this.unpackValue() - - // Convert the msgpack value back to bytes - val tempPacker = MessagePack.newDefaultBufferPacker() - tempPacker.packValue(msgpackValue) - val msgpackBytes = tempPacker.toByteArray() - tempPacker.close() - - // Let Jackson deserialize the msgpack bytes back to ObjectMessage +internal fun ImmutableMapValue.toObjectMessage(): ObjectMessage { + val msgpackBytes = MessagePack.newDefaultBufferPacker().use { packer -> + packer.packValue(this) + packer.toByteArray() + } return msgpackMapper.readValue(msgpackBytes, ObjectMessage::class.java) } @@ -66,7 +60,7 @@ internal class DefaultLiveObjectSerializer : LiveObjectSerializer { override fun readMsgpackArray(unpacker: MessageUnpacker): Array { val objectMessagesCount = unpacker.unpackArrayHeader() - return Array(objectMessagesCount) { unpacker.readObjectMessage() } + return Array(objectMessagesCount) { unpacker.unpackValue().asMapValue().toObjectMessage() } } override fun writeMsgpackArray(objects: Array?, packer: MessagePacker) { @@ -101,7 +95,7 @@ internal class ObjectDataJsonSerializer : JsonSerializer, JsonDeseri when (val v = value.value) { is Boolean -> obj.addProperty("boolean", v) is String -> obj.addProperty("string", v) - is Number -> obj.addProperty("number", v) + is Number -> obj.addProperty("number", v.toDouble()) is Binary -> obj.addProperty("bytes", Base64.getEncoder().encodeToString(v.data)) // Spec: OD4c5 is JsonObject, is JsonArray -> { @@ -132,7 +126,7 @@ internal class ObjectDataJsonSerializer : JsonSerializer, JsonDeseri ) } obj.has("string") -> ObjectValue(obj.get("string").asString) - obj.has("number") -> ObjectValue(obj.get("number").asNumber) + obj.has("number") -> ObjectValue(obj.get("number").asDouble) obj.has("bytes") -> ObjectValue(Binary(Base64.getDecoder().decode(obj.get("bytes").asString))) else -> throw JsonParseException("ObjectData must have one of the fields: boolean, string, number, or bytes") } @@ -179,10 +173,32 @@ internal class ObjectDataMsgpackDeserializer : com.fasterxml.jackson.databind.Js ) } node.has("string") -> ObjectValue(node.get("string").asText()) - node.has("number") -> ObjectValue(node.get("number").numberValue()) + node.has("number") -> ObjectValue(node.get("number").doubleValue()) node.has("bytes") -> ObjectValue(Binary(node.get("bytes").binaryValue())) else -> throw IllegalArgumentException("ObjectData must have one of the fields: boolean, string, number, or bytes") } return ObjectData(objectId, value) } } + +internal class InitialValueJsonSerializer : JsonSerializer, JsonDeserializer { + override fun serialize(src: Binary, typeOfSrc: Type?, context: JsonSerializationContext?): JsonElement { + return JsonPrimitive(Base64.getEncoder().encodeToString(src.data)) + } + + override fun deserialize(json: JsonElement, typeOfT: Type?, context: JsonDeserializationContext?): Binary { + return Binary(Base64.getDecoder().decode(json.asString)) + } +} + +internal class InitialValueMsgpackSerializer : com.fasterxml.jackson.databind.JsonSerializer() { + override fun serialize(value: Binary?, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeBinary(value?.data) + } +} + +internal class InitialValueMsgpackDeserializer : com.fasterxml.jackson.databind.JsonDeserializer() { + override fun deserialize(p: com.fasterxml.jackson.core.JsonParser, ctxt: DeserializationContext): Binary { + return Binary(p.binaryValue) + } +} diff --git a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt new file mode 100644 index 000000000..af4785435 --- /dev/null +++ b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt @@ -0,0 +1,78 @@ +package io.ably.lib.objects.unit + +import io.ably.lib.objects.gson +import io.ably.lib.objects.msgpackMapper +import io.ably.lib.objects.unit.fixtures.* +import io.ably.lib.types.ProtocolMessage +import io.ably.lib.types.ProtocolSerializer +import io.ably.lib.util.Serialisation +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Test +import kotlin.test.assertNotNull + +class ObjectMessageSerializationTest { + + private val objectMessages = arrayOf( + dummyObjectMessageWithStringData(), + dummyObjectMessageWithBinaryData(), + dummyObjectMessageWithNumberData(), + dummyObjectMessageWithBooleanData(), + dummyObjectMessageWithJsonObjectData(), + dummyObjectMessageWithJsonArrayData() + ) + + @Test + fun testObjectMessageMsgPackSerialization() = runTest { + val protocolMessage = ProtocolMessage() + protocolMessage.action = ProtocolMessage.Action.`object` + protocolMessage.state = objectMessages + + // Serialize the ProtocolMessage containing ObjectMessages to MsgPack format + val serializedProtoMsg = ProtocolSerializer.writeMsgpack(protocolMessage) + assertNotNull(serializedProtoMsg) + + // Deserialize back to ProtocolMessage + val deserializedProtoMsg = ProtocolSerializer.readMsgpack(serializedProtoMsg) + assertNotNull(deserializedProtoMsg) + + deserializedProtoMsg.state.zip(objectMessages).forEach { (actual, expected) -> + assertEquals(expected, actual as? io.ably.lib.objects.ObjectMessage) + } + } + + @Test + fun testObjectMessageJsonSerialization() = runTest { + val protocolMessage = ProtocolMessage() + protocolMessage.action = ProtocolMessage.Action.`object` + protocolMessage.state = objectMessages + + // Serialize the ProtocolMessage containing ObjectMessages to MsgPack format + val serializedProtoMsg = ProtocolSerializer.writeJSON(protocolMessage).toString(Charsets.UTF_8) + assertNotNull(serializedProtoMsg) + + // Deserialize back to ProtocolMessage + val deserializedProtoMsg = ProtocolSerializer.fromJSON(serializedProtoMsg) + assertNotNull(deserializedProtoMsg) + + deserializedProtoMsg.state.zip(objectMessages).forEach { (actual, expected) -> + assertEquals(expected, (actual as? io.ably.lib.objects.ObjectMessage)) + } + } + + @Test + fun testOmitNullInSerialization() = runTest { + val nullableObject = object { + val name = "Test Object" + val description: String? = null // This will be omitted if using Gson with excludeNulls + val value = 42 + } + val serializedJsonString = gson.toJson(nullableObject) + // check serializedObject does not contain the null field + assertEquals("""{"name":"Test Object","value":42}""", serializedJsonString) + + val serializedMsgpackBytes = msgpackMapper.writeValueAsBytes(nullableObject) + // check serializedObject does not contain the null field + assertEquals("""{"name":"Test Object","value":42}""", Serialisation.msgpackToGson(serializedMsgpackBytes).toString()) + } +} diff --git a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt index 9de77219f..524921fe6 100644 --- a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt +++ b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt @@ -11,12 +11,14 @@ import io.ably.lib.objects.ensureMessageSizeWithinLimit import io.ably.lib.objects.size import io.ably.lib.transport.Defaults import io.ably.lib.types.AblyException +import io.ktor.utils.io.core.* import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.test.runTest import org.junit.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.text.toByteArray class ObjectMessageSizeTest { @@ -79,7 +81,7 @@ class ObjectMessageSizeTest { ), // Total ObjectCounter size: 8 bytes nonce = "nonce123", // Not counted in operation size - initialValue = "some-value", // Not counted in operation size + initialValue = Binary("some-value".toByteArray()), // Not counted in operation size ), // Total ObjectOperation size: 12 + 8 + 26 + 8 = 54 bytes objectState = ObjectState( diff --git a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt new file mode 100644 index 000000000..2c2738c4c --- /dev/null +++ b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt @@ -0,0 +1,174 @@ +package io.ably.lib.objects.unit.fixtures + +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import io.ably.lib.objects.* +import io.ably.lib.objects.Binary +import io.ably.lib.objects.ObjectData +import io.ably.lib.objects.ObjectMessage +import io.ably.lib.objects.ObjectState +import io.ably.lib.objects.ObjectValue + +internal val dummyObjectDataStringValue = ObjectData(objectId = "object-id", ObjectValue("dummy string")) + +internal val dummyBinaryObjectValue = ObjectData(objectId = "object-id", ObjectValue(Binary(byteArrayOf(1, 2, 3)))) + +internal val dummyNumberObjectValue = ObjectData(objectId = "object-id", ObjectValue(42.0)) + +internal val dummyBooleanObjectValue = ObjectData(objectId = "object-id", ObjectValue(true)) + +val dummyJsonObject = JsonObject().apply { addProperty("foo", "bar") } +internal val dummyJsonObjectValue = ObjectData(objectId = "object-id", ObjectValue(dummyJsonObject)) + +val dummyJsonArray = JsonArray().apply { add(1); add(2); add(3) } +internal val dummyJsonArrayValue = ObjectData(objectId = "object-id", ObjectValue(dummyJsonArray)) + +internal val dummyObjectMapEntry = ObjectMapEntry( + tombstone = false, + timeserial = "dummy-timeserial", + data = dummyObjectDataStringValue +) + +internal val dummyObjectMap = ObjectMap( + semantics = MapSemantics.LWW, + entries = mapOf("dummy-key" to dummyObjectMapEntry) +) + +internal val dummyObjectCounter = ObjectCounter( + count = 123.0 +) + +internal val dummyObjectMapOp = ObjectMapOp( + key = "dummy-key", + data = dummyObjectDataStringValue +) + +internal val dummyObjectCounterOp = ObjectCounterOp( + amount = 10.0 +) + +internal val dummyObjectOperation = ObjectOperation( + action = ObjectOperationAction.MapCreate, + objectId = "dummy-object-id", + mapOp = dummyObjectMapOp, + counterOp = dummyObjectCounterOp, + map = dummyObjectMap, + counter = dummyObjectCounter, + nonce = "dummy-nonce", + initialValue = Binary("{\"foo\":\"bar\"}".toByteArray()) +) + +internal val dummyObjectState = ObjectState( + objectId = "dummy-object-id", + siteTimeserials = mapOf("site1" to "serial1"), + tombstone = false, + createOp = dummyObjectOperation, + map = dummyObjectMap, + counter = dummyObjectCounter +) + +internal val dummyObjectMessage = ObjectMessage( + id = "dummy-id", + timestamp = 1234567890L, + clientId = "dummy-client-id", + connectionId = "dummy-connection-id", + extras = mapOf("meta" to "data"), + operation = dummyObjectOperation, + objectState = dummyObjectState, + serial = "dummy-serial", + siteCode = "dummy-site-code" +) + +internal fun dummyObjectMessageWithStringData(): ObjectMessage { + return dummyObjectMessage +} + +internal fun dummyObjectMessageWithBinaryData(): ObjectMessage { + val binaryObjectMapEntry = dummyObjectMapEntry.copy(data = dummyBinaryObjectValue) + val binaryObjectMap = dummyObjectMap.copy(entries = mapOf("dummy-key" to binaryObjectMapEntry)) + val binaryObjectMapOp = dummyObjectMapOp.copy(data = dummyBinaryObjectValue) + val binaryObjectOperation = dummyObjectOperation.copy( + mapOp = binaryObjectMapOp, + map = binaryObjectMap + ) + val binaryObjectState = dummyObjectState.copy( + map = binaryObjectMap, + createOp = binaryObjectOperation + ) + return dummyObjectMessage.copy( + operation = binaryObjectOperation, + objectState = binaryObjectState + ) +} + +internal fun dummyObjectMessageWithNumberData(): ObjectMessage { + val numberObjectMapEntry = dummyObjectMapEntry.copy(data = dummyNumberObjectValue) + val numberObjectMap = dummyObjectMap.copy(entries = mapOf("dummy-key" to numberObjectMapEntry)) + val numberObjectMapOp = dummyObjectMapOp.copy(data = dummyNumberObjectValue) + val numberObjectOperation = dummyObjectOperation.copy( + mapOp = numberObjectMapOp, + map = numberObjectMap + ) + val numberObjectState = dummyObjectState.copy( + map = numberObjectMap, + createOp = numberObjectOperation + ) + return dummyObjectMessage.copy( + operation = numberObjectOperation, + objectState = numberObjectState + ) +} + +internal fun dummyObjectMessageWithBooleanData(): ObjectMessage { + val booleanObjectMapEntry = dummyObjectMapEntry.copy(data = dummyBooleanObjectValue) + val booleanObjectMap = dummyObjectMap.copy(entries = mapOf("dummy-key" to booleanObjectMapEntry)) + val booleanObjectMapOp = dummyObjectMapOp.copy(data = dummyBooleanObjectValue) + val booleanObjectOperation = dummyObjectOperation.copy( + mapOp = booleanObjectMapOp, + map = booleanObjectMap + ) + val booleanObjectState = dummyObjectState.copy( + map = booleanObjectMap, + createOp = booleanObjectOperation + ) + return dummyObjectMessage.copy( + operation = booleanObjectOperation, + objectState = booleanObjectState + ) +} + +internal fun dummyObjectMessageWithJsonObjectData(): ObjectMessage { + val jsonObjectMapEntry = dummyObjectMapEntry.copy(data = dummyJsonObjectValue) + val jsonObjectMap = dummyObjectMap.copy(entries = mapOf("dummy-key" to jsonObjectMapEntry)) + val jsonObjectMapOp = dummyObjectMapOp.copy(data = dummyJsonObjectValue) + val jsonObjectOperation = dummyObjectOperation.copy( + mapOp = jsonObjectMapOp, + map = jsonObjectMap + ) + val jsonObjectState = dummyObjectState.copy( + map = jsonObjectMap, + createOp = jsonObjectOperation + ) + return dummyObjectMessage.copy( + operation = jsonObjectOperation, + objectState = jsonObjectState + ) +} + +internal fun dummyObjectMessageWithJsonArrayData(): ObjectMessage { + val jsonArrayMapEntry = dummyObjectMapEntry.copy(data = dummyJsonArrayValue) + val jsonArrayMap = dummyObjectMap.copy(entries = mapOf("dummy-key" to jsonArrayMapEntry)) + val jsonArrayMapOp = dummyObjectMapOp.copy(data = dummyJsonArrayValue) + val jsonArrayOperation = dummyObjectOperation.copy( + mapOp = jsonArrayMapOp, + map = jsonArrayMap + ) + val jsonArrayState = dummyObjectState.copy( + map = jsonArrayMap, + createOp = jsonArrayOperation + ) + return dummyObjectMessage.copy( + operation = jsonArrayOperation, + objectState = jsonArrayState + ) +} From 792ffc984443556a0934a51cf5de2c137900fa6b Mon Sep 17 00:00:00 2001 From: sacOO7 Date: Sun, 22 Jun 2025 18:14:56 +0530 Subject: [PATCH 5/5] [ECO-5386] Updated unit tests for ObjectMesssage serialization 1. Updated test for testOmitNullsInObjectMessageSerialization 2. Added test HandleNullsInObjectMessageDeserialization --- gradle/libs.versions.toml | 2 +- .../ably/lib/objects/LiveObjectsHelper.java | 2 +- .../io/ably/lib/types/ProtocolSerializer.java | 15 +- .../kotlin/io/ably/lib/objects/Helpers.kt | 8 +- .../io/ably/lib/objects/ObjectMessage.kt | 12 +- .../io/ably/lib/objects/Serialization.kt | 204 ------------------ .../serialization/JsonSerialization.kt | 99 +++++++++ .../serialization/MsgpackSerialization.kt | 115 ++++++++++ .../objects/serialization/Serialization.kt | 45 ++++ .../unit/ObjectMessageSerializationTest.kt | 128 +++++++++-- .../lib/objects/unit/ObjectMessageSizeTest.kt | 10 +- .../unit/fixtures/ObjectMessageFixture.kt | 2 +- 12 files changed, 405 insertions(+), 237 deletions(-) delete mode 100644 live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt create mode 100644 live-objects/src/main/kotlin/io/ably/lib/objects/serialization/JsonSerialization.kt create mode 100644 live-objects/src/main/kotlin/io/ably/lib/objects/serialization/MsgpackSerialization.kt create mode 100644 live-objects/src/main/kotlin/io/ably/lib/objects/serialization/Serialization.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 84517b90c..554d4715b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -26,7 +26,7 @@ turbine = "1.2.0" ktor = "3.1.3" jetbrains-annoations = "26.0.2" jackson-msgpack = "0.8.11" # Compatible with msgpack-core 0.8.11 -jackson-param = "2.18.0" # Compatible with jackson-msgpack +jackson-param = "2.19.1" # Compatible with jackson-msgpack [libraries] gson = { group = "com.google.code.gson", name = "gson", version.ref = "gson" } diff --git a/lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java b/lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java index 241d5d0d2..d7c48542f 100644 --- a/lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java +++ b/lib/src/main/java/io/ably/lib/objects/LiveObjectsHelper.java @@ -27,7 +27,7 @@ public static LiveObjectsPlugin tryInitializeLiveObjectsPlugin(AblyRealtime ably public static LiveObjectSerializer getLiveObjectSerializer() { if (liveObjectSerializer == null) { try { - Class serializerClass = Class.forName("io.ably.lib.objects.DefaultLiveObjectSerializer"); + Class serializerClass = Class.forName("io.ably.lib.objects.serialization.DefaultLiveObjectSerializer"); liveObjectSerializer = (LiveObjectSerializer) serializerClass.getDeclaredConstructor().newInstance(); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { diff --git a/lib/src/main/java/io/ably/lib/types/ProtocolSerializer.java b/lib/src/main/java/io/ably/lib/types/ProtocolSerializer.java index 97e5fc80b..e33bfe186 100644 --- a/lib/src/main/java/io/ably/lib/types/ProtocolSerializer.java +++ b/lib/src/main/java/io/ably/lib/types/ProtocolSerializer.java @@ -14,7 +14,7 @@ public class ProtocolSerializer { /**************************************** * Msgpack decode ****************************************/ - + public static ProtocolMessage readMsgpack(byte[] packed) throws AblyException { try { MessageUnpacker unpacker = Serialisation.msgpackUnpackerConfig.newUnpacker(packed); @@ -27,22 +27,23 @@ public static ProtocolMessage readMsgpack(byte[] packed) throws AblyException { /**************************************** * Msgpack encode ****************************************/ - - public static byte[] writeMsgpack(ProtocolMessage message) { + + public static byte[] writeMsgpack(ProtocolMessage message) throws AblyException { ByteArrayOutputStream out = new ByteArrayOutputStream(); MessagePacker packer = Serialisation.msgpackPackerConfig.newPacker(out); try { message.writeMsgpack(packer); - packer.flush(); return out.toByteArray(); - } catch(IOException e) { return null; } + } catch (IOException ioe) { + throw AblyException.fromThrowable(ioe); + } } /**************************************** * JSON decode ****************************************/ - + public static ProtocolMessage fromJSON(String packed) throws AblyException { return Serialisation.gson.fromJson(packed, ProtocolMessage.class); } @@ -50,7 +51,7 @@ public static ProtocolMessage fromJSON(String packed) throws AblyException { /**************************************** * JSON encode ****************************************/ - + public static byte[] writeJSON(ProtocolMessage message) throws AblyException { return Serialisation.gson.toJson(message).getBytes(Charset.forName("UTF-8")); } diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/Helpers.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/Helpers.kt index 3da94183b..be6373eae 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/Helpers.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/Helpers.kt @@ -39,18 +39,18 @@ internal enum class ProtocolMessageFormat(private val value: String) { override fun toString(): String = value } -internal class Binary(val data: ByteArray?) { +internal class Binary(val data: ByteArray) { override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Binary) return false - return data?.contentEquals(other.data) == true + return data.contentEquals(other.data) } override fun hashCode(): Int { - return data?.contentHashCode() ?: 0 + return data.contentHashCode() } } internal fun Binary.size(): Int { - return data?.size ?: 0 + return data.size } diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt index 1a694a132..61d3a270d 100644 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/ObjectMessage.kt @@ -8,6 +8,14 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.annotation.JsonSerialize import com.google.gson.annotations.JsonAdapter import com.google.gson.annotations.SerializedName +import io.ably.lib.objects.serialization.* +import io.ably.lib.objects.serialization.InitialValueJsonSerializer +import io.ably.lib.objects.serialization.InitialValueMsgpackDeserializer +import io.ably.lib.objects.serialization.InitialValueMsgpackSerializer +import io.ably.lib.objects.serialization.ObjectDataJsonSerializer +import io.ably.lib.objects.serialization.ObjectDataMsgpackDeserializer +import io.ably.lib.objects.serialization.ObjectDataMsgpackSerializer +import io.ably.lib.objects.serialization.gson /** * An enum class representing the different actions that can be performed on an object. @@ -319,7 +327,9 @@ internal data class ObjectMessage( * or validation of the @extras@ field itself, but should treat it opaquely, encoding it and passing it to realtime unaltered * Spec: OM2d */ - val extras: Any? = null, + @JsonSerialize(using = JsonObjectMsgpackSerializer::class) + @JsonDeserialize(using = JsonObjectMsgpackDeserializer::class) + val extras: JsonObject? = null, /** * Describes an operation to be applied to an object. diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt deleted file mode 100644 index 319cb33b8..000000000 --- a/live-objects/src/main/kotlin/io/ably/lib/objects/Serialization.kt +++ /dev/null @@ -1,204 +0,0 @@ -@file:Suppress("UNCHECKED_CAST") - -package io.ably.lib.objects - -import com.fasterxml.jackson.annotation.JsonCreator -import com.fasterxml.jackson.annotation.JsonInclude -import com.fasterxml.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.DeserializationContext -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.SerializerProvider -import com.fasterxml.jackson.module.paramnames.ParameterNamesModule -import com.google.gson.* -import org.msgpack.core.MessagePack -import org.msgpack.core.MessagePacker -import org.msgpack.core.MessageUnpacker -import org.msgpack.jackson.dataformat.MessagePackFactory -import org.msgpack.value.ImmutableMapValue -import java.lang.reflect.Type -import java.util.* - -// Gson instance for JSON serialization/deserialization -internal val gson: Gson = GsonBuilder().create() - -// Jackson ObjectMapper for MessagePack serialization (respects @JsonProperty annotations) -// Caches type metadata and serializers for ObjectMessage class after first use, so next time it's super fast 🚀 -// https://github.com/FasterXML/jackson-modules-java8/tree/3.x/parameter-names -internal val msgpackMapper = ObjectMapper(MessagePackFactory()).apply { - registerModule(ParameterNamesModule(JsonCreator.Mode.PROPERTIES)) - setSerializationInclusion(JsonInclude.Include.NON_NULL) -} - -internal fun ObjectMessage.toJsonObject(): JsonObject { - return gson.toJsonTree(this).asJsonObject -} - -internal fun JsonObject.toObjectMessage(): ObjectMessage { - return gson.fromJson(this, ObjectMessage::class.java) -} - -internal fun ObjectMessage.writeTo(packer: MessagePacker) { - val msgpackBytes = msgpackMapper.writeValueAsBytes(this) // returns correct msgpack map structure - packer.writePayload(msgpackBytes) -} - -internal fun ImmutableMapValue.toObjectMessage(): ObjectMessage { - val msgpackBytes = MessagePack.newDefaultBufferPacker().use { packer -> - packer.packValue(this) - packer.toByteArray() - } - return msgpackMapper.readValue(msgpackBytes, ObjectMessage::class.java) -} - -/** - * Default implementation of {@link LiveObjectSerializer} that handles serialization/deserialization - * of ObjectMessage arrays for both JSON and MessagePack formats using Jackson and Gson. - * Dynamically loaded by LiveObjectsHelper#getLiveObjectSerializer() to avoid hard dependencies. - */ -@Suppress("unused") // Used via reflection in LiveObjectsHelper -internal class DefaultLiveObjectSerializer : LiveObjectSerializer { - - override fun readMsgpackArray(unpacker: MessageUnpacker): Array { - val objectMessagesCount = unpacker.unpackArrayHeader() - return Array(objectMessagesCount) { unpacker.unpackValue().asMapValue().toObjectMessage() } - } - - override fun writeMsgpackArray(objects: Array?, packer: MessagePacker) { - val objectMessages: Array = objects as Array - packer.packArrayHeader(objectMessages.size) - objectMessages.forEach { it.writeTo(packer) } - } - - override fun readFromJsonArray(json: JsonArray): Array { - return json.map { element -> - if (element.isJsonObject) element.asJsonObject.toObjectMessage() - else throw JsonParseException("Expected JsonObject, but found: $element") - }.toTypedArray() - } - - override fun asJsonArray(objects: Array?): JsonArray { - val objectMessages: Array = objects as Array - val jsonArray = JsonArray() - for (objectMessage in objectMessages) { - jsonArray.add(objectMessage.toJsonObject()) - } - return jsonArray - } -} - -internal class ObjectDataJsonSerializer : JsonSerializer, JsonDeserializer { - override fun serialize(src: ObjectData?, typeOfSrc: Type?, context: JsonSerializationContext?): JsonElement { - val obj = JsonObject() - src?.objectId?.let { obj.addProperty("objectId", it) } - - src?.value?.let { value -> - when (val v = value.value) { - is Boolean -> obj.addProperty("boolean", v) - is String -> obj.addProperty("string", v) - is Number -> obj.addProperty("number", v.toDouble()) - is Binary -> obj.addProperty("bytes", Base64.getEncoder().encodeToString(v.data)) - // Spec: OD4c5 - is JsonObject, is JsonArray -> { - obj.addProperty("string", v.toString()) - obj.addProperty("encoding", "json") - } - } - } - return obj - } - - override fun deserialize(json: JsonElement?, typeOfT: Type?, context: JsonDeserializationContext?): ObjectData { - val obj = if (json?.isJsonObject == true) json.asJsonObject else throw JsonParseException("Expected JsonObject") - val objectId = if (obj.has("objectId")) obj.get("objectId").asString else null - val encoding = if (obj.has("encoding")) obj.get("encoding").asString else null - val value = when { - obj.has("boolean") -> ObjectValue(obj.get("boolean").asBoolean) - // Spec: OD5b3 - obj.has("string") && encoding == "json" -> { - val jsonStr = obj.get("string").asString - val parsed = JsonParser.parseString(jsonStr) - ObjectValue( - when { - parsed.isJsonObject -> parsed.asJsonObject - parsed.isJsonArray -> parsed.asJsonArray - else -> throw JsonParseException("Invalid JSON string for encoding=json") - } - ) - } - obj.has("string") -> ObjectValue(obj.get("string").asString) - obj.has("number") -> ObjectValue(obj.get("number").asDouble) - obj.has("bytes") -> ObjectValue(Binary(Base64.getDecoder().decode(obj.get("bytes").asString))) - else -> throw JsonParseException("ObjectData must have one of the fields: boolean, string, number, or bytes") - } - return ObjectData(objectId, value) - } -} - -internal class ObjectDataMsgpackSerializer : com.fasterxml.jackson.databind.JsonSerializer() { - override fun serialize(value: ObjectData?, gen: JsonGenerator, serializers: SerializerProvider) { - gen.writeStartObject() - value?.objectId?.let { gen.writeStringField("objectId", it) } - value?.value?.let { v -> - when (val data = v.value) { - is Boolean -> gen.writeBooleanField("boolean", data) - is String -> gen.writeStringField("string", data) - is Number -> gen.writeNumberField("number", data.toDouble()) - is Binary -> gen.writeBinaryField("bytes", data.data) - is JsonObject, is JsonArray -> { - gen.writeStringField("string", data.toString()) - gen.writeStringField("encoding", "json") - } - } - } - gen.writeEndObject() - } -} - -internal class ObjectDataMsgpackDeserializer : com.fasterxml.jackson.databind.JsonDeserializer() { - override fun deserialize(p: com.fasterxml.jackson.core.JsonParser, ctxt: DeserializationContext): ObjectData { - val node = p.codec.readTree(p) - val objectId = node.get("objectId")?.asText() - val encoding = node.get("encoding")?.asText() - val value = when { - node.has("boolean") -> ObjectValue(node.get("boolean").asBoolean()) - node.has("string") && encoding == "json" -> { - val jsonStr = node.get("string").asText() - val parsed = JsonParser.parseString(jsonStr) - ObjectValue( - when { - parsed.isJsonObject -> parsed.asJsonObject - parsed.isJsonArray -> parsed.asJsonArray - else -> throw IllegalArgumentException("Invalid JSON string for encoding=json") - } - ) - } - node.has("string") -> ObjectValue(node.get("string").asText()) - node.has("number") -> ObjectValue(node.get("number").doubleValue()) - node.has("bytes") -> ObjectValue(Binary(node.get("bytes").binaryValue())) - else -> throw IllegalArgumentException("ObjectData must have one of the fields: boolean, string, number, or bytes") - } - return ObjectData(objectId, value) - } -} - -internal class InitialValueJsonSerializer : JsonSerializer, JsonDeserializer { - override fun serialize(src: Binary, typeOfSrc: Type?, context: JsonSerializationContext?): JsonElement { - return JsonPrimitive(Base64.getEncoder().encodeToString(src.data)) - } - - override fun deserialize(json: JsonElement, typeOfT: Type?, context: JsonDeserializationContext?): Binary { - return Binary(Base64.getDecoder().decode(json.asString)) - } -} - -internal class InitialValueMsgpackSerializer : com.fasterxml.jackson.databind.JsonSerializer() { - override fun serialize(value: Binary?, gen: JsonGenerator, serializers: SerializerProvider) { - gen.writeBinary(value?.data) - } -} - -internal class InitialValueMsgpackDeserializer : com.fasterxml.jackson.databind.JsonDeserializer() { - override fun deserialize(p: com.fasterxml.jackson.core.JsonParser, ctxt: DeserializationContext): Binary { - return Binary(p.binaryValue) - } -} diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/JsonSerialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/JsonSerialization.kt new file mode 100644 index 000000000..c8a212845 --- /dev/null +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/JsonSerialization.kt @@ -0,0 +1,99 @@ +package io.ably.lib.objects.serialization + +import com.google.gson.* +import io.ably.lib.objects.* +import io.ably.lib.objects.Binary +import io.ably.lib.objects.ObjectData +import io.ably.lib.objects.ObjectMessage +import io.ably.lib.objects.ObjectOperationAction +import io.ably.lib.objects.ObjectValue +import java.lang.reflect.Type +import java.util.* +import kotlin.enums.EnumEntries + +// Gson instance for JSON serialization/deserialization +internal val gson = GsonBuilder() + .registerTypeAdapter(ObjectOperationAction::class.java, EnumCodeTypeAdapter({ it.code }, ObjectOperationAction.entries)) + .registerTypeAdapter(MapSemantics::class.java, EnumCodeTypeAdapter({ it.code }, MapSemantics.entries)) + .create() + +internal fun ObjectMessage.toJsonObject(): JsonObject { + return gson.toJsonTree(this).asJsonObject +} + +internal fun JsonObject.toObjectMessage(): ObjectMessage { + return gson.fromJson(this, ObjectMessage::class.java) +} + +internal class EnumCodeTypeAdapter>( + private val getCode: (T) -> Int, + private val enumValues: EnumEntries +) : JsonSerializer, JsonDeserializer { + + override fun serialize(src: T, typeOfSrc: Type, context: JsonSerializationContext): JsonElement { + return JsonPrimitive(getCode(src)) + } + + override fun deserialize(json: JsonElement, typeOfT: Type, context: JsonDeserializationContext): T { + val code = json.asInt + return enumValues.first { getCode(it) == code } + } +} + +internal class ObjectDataJsonSerializer : JsonSerializer, JsonDeserializer { + override fun serialize(src: ObjectData, typeOfSrc: Type?, context: JsonSerializationContext?): JsonElement { + val obj = JsonObject() + src.objectId?.let { obj.addProperty("objectId", it) } + + src.value?.let { value -> + when (val v = value.value) { + is Boolean -> obj.addProperty("boolean", v) + is String -> obj.addProperty("string", v) + is Number -> obj.addProperty("number", v.toDouble()) + is Binary -> obj.addProperty("bytes", Base64.getEncoder().encodeToString(v.data)) + // Spec: OD4c5 + is JsonObject, is JsonArray -> { + obj.addProperty("string", v.toString()) + obj.addProperty("encoding", "json") + } + } + } + return obj + } + + override fun deserialize(json: JsonElement, typeOfT: Type?, context: JsonDeserializationContext?): ObjectData { + val obj = if (json.isJsonObject) json.asJsonObject else throw JsonParseException("Expected JsonObject") + val objectId = if (obj.has("objectId")) obj.get("objectId").asString else null + val encoding = if (obj.has("encoding")) obj.get("encoding").asString else null + val value = when { + obj.has("boolean") -> ObjectValue(obj.get("boolean").asBoolean) + // Spec: OD5b3 + obj.has("string") && encoding == "json" -> { + val jsonStr = obj.get("string").asString + val parsed = JsonParser.parseString(jsonStr) + ObjectValue( + when { + parsed.isJsonObject -> parsed.asJsonObject + parsed.isJsonArray -> parsed.asJsonArray + else -> throw JsonParseException("Invalid JSON string for encoding=json") + } + ) + } + obj.has("string") -> ObjectValue(obj.get("string").asString) + obj.has("number") -> ObjectValue(obj.get("number").asDouble) + obj.has("bytes") -> ObjectValue(Binary(Base64.getDecoder().decode(obj.get("bytes").asString))) + else -> throw JsonParseException("ObjectData must have one of the fields: boolean, string, number, or bytes") + } + return ObjectData(objectId, value) + } +} + +internal class InitialValueJsonSerializer : JsonSerializer, JsonDeserializer { + override fun serialize(src: Binary, typeOfSrc: Type?, context: JsonSerializationContext?): JsonElement { + return JsonPrimitive(Base64.getEncoder().encodeToString(src.data)) + } + + override fun deserialize(json: JsonElement, typeOfT: Type?, context: JsonDeserializationContext?): Binary { + return Binary(Base64.getDecoder().decode(json.asString)) + } +} diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/MsgpackSerialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/MsgpackSerialization.kt new file mode 100644 index 000000000..32235e59b --- /dev/null +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/MsgpackSerialization.kt @@ -0,0 +1,115 @@ +package io.ably.lib.objects.serialization + +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import io.ably.lib.objects.Binary +import io.ably.lib.objects.ObjectData +import io.ably.lib.objects.ObjectMessage +import io.ably.lib.objects.ObjectValue +import io.ably.lib.util.Serialisation +import org.msgpack.core.MessagePack +import org.msgpack.core.MessagePacker +import org.msgpack.jackson.dataformat.MessagePackFactory +import org.msgpack.value.ImmutableMapValue + +// Jackson ObjectMapper for MessagePack serialization (respects @JsonProperty annotations) +// Caches type metadata and serializers for ObjectMessage class after first use, so next time it's super fast 🚀 +// https://github.com/FasterXML/jackson-modules-java8/tree/3.x/parameter-names +internal val msgpackMapper = ObjectMapper(MessagePackFactory()).apply { + registerModule(ParameterNamesModule(JsonCreator.Mode.PROPERTIES)) + setSerializationInclusion(JsonInclude.Include.NON_NULL) + enable(SerializationFeature.WRITE_ENUMS_USING_INDEX) // Serialize enums using their ordinal values +} + +internal fun ObjectMessage.writeTo(packer: MessagePacker) { + val msgpackBytes = msgpackMapper.writeValueAsBytes(this) // returns correct msgpack map structure + packer.writePayload(msgpackBytes) +} + +internal fun ImmutableMapValue.toObjectMessage(): ObjectMessage { + val msgpackBytes = MessagePack.newDefaultBufferPacker().use { packer -> + packer.packValue(this) + packer.toByteArray() + } + return msgpackMapper.readValue(msgpackBytes, ObjectMessage::class.java) +} + +internal class ObjectDataMsgpackSerializer : com.fasterxml.jackson.databind.JsonSerializer() { + override fun serialize(value: ObjectData, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeStartObject() + value.objectId?.let { gen.writeStringField("objectId", it) } + value.value?.let { v -> + when (val data = v.value) { + is Boolean -> gen.writeBooleanField("boolean", data) + is String -> gen.writeStringField("string", data) + is Number -> gen.writeNumberField("number", data.toDouble()) + is Binary -> gen.writeBinaryField("bytes", data.data) + is JsonObject, is JsonArray -> { + gen.writeStringField("string", data.toString()) + gen.writeStringField("encoding", "json") + } + } + } + gen.writeEndObject() + } +} + +internal class ObjectDataMsgpackDeserializer : com.fasterxml.jackson.databind.JsonDeserializer() { + override fun deserialize(p: com.fasterxml.jackson.core.JsonParser, ctxt: DeserializationContext): ObjectData { + val node = p.codec.readTree(p) + val objectId = node.get("objectId")?.asText() + val encoding = node.get("encoding")?.asText() + val value = when { + node.has("boolean") -> ObjectValue(node.get("boolean").asBoolean()) + node.has("string") && encoding == "json" -> { + val jsonStr = node.get("string").asText() + val parsed = JsonParser.parseString(jsonStr) + ObjectValue( + when { + parsed.isJsonObject -> parsed.asJsonObject + parsed.isJsonArray -> parsed.asJsonArray + else -> throw IllegalArgumentException("Invalid JSON string for encoding=json") + } + ) + } + node.has("string") -> ObjectValue(node.get("string").asText()) + node.has("number") -> ObjectValue(node.get("number").doubleValue()) + node.has("bytes") -> ObjectValue(Binary(node.get("bytes").binaryValue())) + else -> throw IllegalArgumentException("ObjectData must have one of the fields: boolean, string, number, or bytes") + } + return ObjectData(objectId, value) + } +} + +internal class InitialValueMsgpackSerializer : com.fasterxml.jackson.databind.JsonSerializer() { + override fun serialize(value: Binary, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeBinary(value.data) + } +} + +internal class InitialValueMsgpackDeserializer : com.fasterxml.jackson.databind.JsonDeserializer() { + override fun deserialize(p: com.fasterxml.jackson.core.JsonParser, ctxt: DeserializationContext): Binary { + return Binary(p.binaryValue) + } +} + +internal class JsonObjectMsgpackSerializer : com.fasterxml.jackson.databind.JsonSerializer() { + override fun serialize(value: JsonObject, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeBinary(Serialisation.gsonToMsgpack(value)) + } +} + +internal class JsonObjectMsgpackDeserializer : com.fasterxml.jackson.databind.JsonDeserializer() { + override fun deserialize(p: com.fasterxml.jackson.core.JsonParser, ctxt: DeserializationContext): JsonObject { + return Serialisation.msgpackToGson(p.binaryValue) as JsonObject + } +} diff --git a/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/Serialization.kt b/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/Serialization.kt new file mode 100644 index 000000000..397c1f5b0 --- /dev/null +++ b/live-objects/src/main/kotlin/io/ably/lib/objects/serialization/Serialization.kt @@ -0,0 +1,45 @@ +@file:Suppress("UNCHECKED_CAST") + +package io.ably.lib.objects.serialization + +import com.google.gson.* +import io.ably.lib.objects.* +import io.ably.lib.objects.ObjectMessage +import org.msgpack.core.MessagePacker +import org.msgpack.core.MessageUnpacker + +/** + * Default implementation of {@link LiveObjectSerializer} that handles serialization/deserialization + * of ObjectMessage arrays for both JSON and MessagePack formats using Jackson and Gson. + * Dynamically loaded by LiveObjectsHelper#getLiveObjectSerializer() to avoid hard dependencies. + */ +@Suppress("unused") // Used via reflection in LiveObjectsHelper +internal class DefaultLiveObjectSerializer : LiveObjectSerializer { + + override fun readMsgpackArray(unpacker: MessageUnpacker): Array { + val objectMessagesCount = unpacker.unpackArrayHeader() + return Array(objectMessagesCount) { unpacker.unpackValue().asMapValue().toObjectMessage() } + } + + override fun writeMsgpackArray(objects: Array?, packer: MessagePacker) { + val objectMessages: Array = objects as Array + packer.packArrayHeader(objectMessages.size) + objectMessages.forEach { it.writeTo(packer) } + } + + override fun readFromJsonArray(json: JsonArray): Array { + return json.map { element -> + if (element.isJsonObject) element.asJsonObject.toObjectMessage() + else throw JsonParseException("Expected JsonObject, but found: $element") + }.toTypedArray() + } + + override fun asJsonArray(objects: Array?): JsonArray { + val objectMessages: Array = objects as Array + val jsonArray = JsonArray() + for (objectMessage in objectMessages) { + jsonArray.add(objectMessage.toJsonObject()) + } + return jsonArray + } +} diff --git a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt index af4785435..2b832388e 100644 --- a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt +++ b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSerializationTest.kt @@ -1,15 +1,20 @@ package io.ably.lib.objects.unit -import io.ably.lib.objects.gson -import io.ably.lib.objects.msgpackMapper +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import com.google.gson.JsonElement +import com.google.gson.JsonNull import io.ably.lib.objects.unit.fixtures.* import io.ably.lib.types.ProtocolMessage +import io.ably.lib.types.ProtocolMessage.ActionSerializer import io.ably.lib.types.ProtocolSerializer import io.ably.lib.util.Serialisation import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals import org.junit.Test import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue class ObjectMessageSerializationTest { @@ -61,18 +66,115 @@ class ObjectMessageSerializationTest { } @Test - fun testOmitNullInSerialization() = runTest { - val nullableObject = object { - val name = "Test Object" - val description: String? = null // This will be omitted if using Gson with excludeNulls - val value = 42 + fun testOmitNullsInObjectMessageSerialization() = runTest { + val objectMessage = dummyObjectMessageWithStringData() + val objectMessageWithNullFields = objectMessage.copy( + id = null, + timestamp = null, + clientId = "test-client", + connectionId = "test-connection", + extras = null, + operation = null, + objectState = null, + serial = null, + siteCode = null + ) + val protocolMessage = ProtocolMessage() + protocolMessage.action = ProtocolMessage.Action.`object` + protocolMessage.state = arrayOf(objectMessageWithNullFields) + + // check if Gson/Msgpack serialization omits null fields + fun assertSerializedObjectMessage(serializedProtoMsg: String) { + val deserializedProtoMsg = Gson().fromJson(serializedProtoMsg, JsonElement::class.java).asJsonObject + val serializedObjectMessage = deserializedProtoMsg.get("state").asJsonArray[0].asJsonObject.toString() + assertEquals("""{"clientId":"test-client","connectionId":"test-connection"}""", serializedObjectMessage) + } + + // Serialize using Gson + val serializedProtoMsg = ProtocolSerializer.writeJSON(protocolMessage).toString(Charsets.UTF_8) + assertSerializedObjectMessage(serializedProtoMsg) + + // Serialize using MsgPack + val serializedMsgpackBytes = ProtocolSerializer.writeMsgpack(protocolMessage) + val serializedJsonStringFromMsgpackBytes = Serialisation.msgpackToGson(serializedMsgpackBytes).toString() + assertSerializedObjectMessage(serializedJsonStringFromMsgpackBytes) + } + + @Test + fun testSerializeEnumsIntoOrdinalValues() = runTest { + val objectMessage = dummyObjectMessageWithStringData() + val protocolMessage = ProtocolMessage() + protocolMessage.action = ProtocolMessage.Action.`object` + protocolMessage.state = arrayOf(objectMessage) + + fun assertSerializedObjectMessage(serializedProtoMsg: String) { + val deserializedProtoMsg = Gson().fromJson(serializedProtoMsg, JsonElement::class.java).asJsonObject + val serializedObjectMessage = deserializedProtoMsg.get("state").asJsonArray[0].asJsonObject + val operation = serializedObjectMessage.get("operation").asJsonObject + assertTrue(operation.has("action")) + assertEquals(0, operation.get("action").asInt) // Check if action is serialized as code } - val serializedJsonString = gson.toJson(nullableObject) - // check serializedObject does not contain the null field - assertEquals("""{"name":"Test Object","value":42}""", serializedJsonString) - val serializedMsgpackBytes = msgpackMapper.writeValueAsBytes(nullableObject) - // check serializedObject does not contain the null field - assertEquals("""{"name":"Test Object","value":42}""", Serialisation.msgpackToGson(serializedMsgpackBytes).toString()) + // Serialize using Gson + val serializedProtoMsg = ProtocolSerializer.writeJSON(protocolMessage).toString(Charsets.UTF_8) + assertSerializedObjectMessage(serializedProtoMsg) + // Serialize using MsgPack + val serializedMsgpackBytes = ProtocolSerializer.writeMsgpack(protocolMessage) + val serializedJsonStringFromMsgpackBytes = Serialisation.msgpackToGson(serializedMsgpackBytes).toString() + assertSerializedObjectMessage(serializedJsonStringFromMsgpackBytes) + } + + @Test + fun testHandleNullsInObjectMessageDeserialization() = runTest { + val protocolMessage = ProtocolMessage() + protocolMessage.id = "id" + protocolMessage.action = ProtocolMessage.Action.`object` + protocolMessage.state = null + + // Serialize using Gson with serializeNulls enabled + val gsonBuilderCreatingNulls = GsonBuilder() + .registerTypeAdapter(ProtocolMessage.Action::class.java, ActionSerializer()) + .serializeNulls().create() + + var protoMsgJsonObject = gsonBuilderCreatingNulls.toJsonTree(protocolMessage).asJsonObject + assertTrue(protoMsgJsonObject.has("state")) + assertEquals(JsonNull.INSTANCE, protoMsgJsonObject.get("state")) + + var deserializedProtoMsg = ProtocolSerializer.fromJSON(protoMsgJsonObject.toString()) + assertNull(deserializedProtoMsg.state) + + var serializedMsgpackBytes = Serialisation.gsonToMsgpack(protoMsgJsonObject) + deserializedProtoMsg = ProtocolSerializer.readMsgpack(serializedMsgpackBytes) + assertNull(deserializedProtoMsg.state) + + // Create ObjectMessage and serialize in a way that resulting string/bytes include null fields + val objectMessage = dummyObjectMessageWithStringData() + val objectMessageWithNullFields = objectMessage.copy( + id = null, + timestamp = null, + clientId = "test-client", + connectionId = "test-connection", + extras = null, + operation = objectMessage.operation?.copy( + initialValue = null, // initialValue set to null + mapOp = objectMessage.operation.mapOp?.copy( + data = null // objectData set to null + ) + ), + objectState = null, + serial = null, + siteCode = null + ) + protocolMessage.state = arrayOf(objectMessageWithNullFields) + protoMsgJsonObject = gsonBuilderCreatingNulls.toJsonTree(protocolMessage).asJsonObject + + // Check if gson deserialization works correctly + deserializedProtoMsg = ProtocolSerializer.fromJSON(protoMsgJsonObject.toString()) + assertEquals(objectMessageWithNullFields, deserializedProtoMsg.state[0] as? io.ably.lib.objects.ObjectMessage) + + // Check if msgpack deserialization works correctly + serializedMsgpackBytes = Serialisation.gsonToMsgpack(protoMsgJsonObject) + deserializedProtoMsg = ProtocolSerializer.readMsgpack(serializedMsgpackBytes) + assertEquals(objectMessageWithNullFields, deserializedProtoMsg.state[0] as? io.ably.lib.objects.ObjectMessage) } } diff --git a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt index 524921fe6..c5fb07248 100644 --- a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt +++ b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/ObjectMessageSizeTest.kt @@ -1,5 +1,6 @@ package io.ably.lib.objects.unit +import com.google.gson.JsonObject import io.ably.lib.objects.* import io.ably.lib.objects.ObjectData import io.ably.lib.objects.ObjectMapOp @@ -11,7 +12,6 @@ import io.ably.lib.objects.ensureMessageSizeWithinLimit import io.ably.lib.objects.size import io.ably.lib.transport.Defaults import io.ably.lib.types.AblyException -import io.ktor.utils.io.core.* import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.test.runTest @@ -34,10 +34,10 @@ class ObjectMessageSizeTest { timestamp = 1699123456789L, // Not counted in size calculation clientId = "test-client", // Size: 11 bytes (UTF-8 byte length) connectionId = "conn_98765", // Not counted in size calculation - extras = mapOf( // Size: JSON serialization byte length - "meta" to "data", // JSON: {"meta":"data","count":42} - "count" to 42 - ), // Total extras size: 26 bytes (verified by gson.toJson().length) + extras = JsonObject().apply { // Size: JSON serialization byte length + addProperty("meta", "data") // JSON: {"meta":"data","count":42} + addProperty("count", 42) + }, // Total extras size: 26 bytes (verified by gson.toJson().length) operation = ObjectOperation( action = ObjectOperationAction.MapCreate, objectId = "obj_54321", // Not counted in operation size diff --git a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt index 2c2738c4c..5821e8dc1 100644 --- a/live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt +++ b/live-objects/src/test/kotlin/io/ably/lib/objects/unit/fixtures/ObjectMessageFixture.kt @@ -72,7 +72,7 @@ internal val dummyObjectMessage = ObjectMessage( timestamp = 1234567890L, clientId = "dummy-client-id", connectionId = "dummy-connection-id", - extras = mapOf("meta" to "data"), + extras = JsonObject().apply { addProperty("meta", "data") }, operation = dummyObjectOperation, objectState = dummyObjectState, serial = "dummy-serial",