From ebfad39c6353a5a2aa1a14a0f16e1d43a93452fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 3 Mar 2025 19:07:26 +0000 Subject: [PATCH 01/58] Add tests for replacing old value after top-level null merges --- tests/unit/onyxTest.ts | 79 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index 44400e715..4ec0b7163 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -1686,7 +1686,86 @@ describe('Onyx', () => { }); }); + it('should replace the old value after a null merge in the top-level object when batching updates', async () => { + let result: unknown; + connection = Onyx.connect({ + key: ONYX_KEYS.COLLECTION.TEST_UPDATE, + waitForCollectionCallback: true, + callback: (value) => { + result = value; + }, + }); + + await Onyx.multiSet({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + id: 'entry1', + someKey: 'someValue', + }, + }); + + const queuedUpdates: OnyxUpdate[] = [ + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + // Removing the entire object in this update. + // Any subsequent changes to this key should completely replace the old value. + value: null, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + // This change should completely replace `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1` old value. + value: { + someKey: 'someValueChanged', + }, + }, + ]; + + await Onyx.update(queuedUpdates); + + expect(result).toEqual({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + someKey: 'someValueChanged', + }, + }); + }); + describe('merge', () => { + it('should replace the old value after a null merge in the top-level object when batching merges', async () => { + let result: unknown; + connection = Onyx.connect({ + key: ONYX_KEYS.COLLECTION.TEST_UPDATE, + waitForCollectionCallback: true, + callback: (value) => { + result = value; + }, + }); + + await Onyx.multiSet({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + id: 'entry1', + someKey: 'someValue', + }, + }); + + // Removing the entire object in this merge. + // Any subsequent changes to this key should completely replace the old value. + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, null); + + // This change should completely replace `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1` old value. + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + someKey: 'someValueChanged', + }); + + await waitForPromisesToResolve(); + + expect(result).toEqual({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + someKey: 'someValueChanged', + }, + }); + }); + it('should remove a deeply nested null when merging an existing key', () => { let result: unknown; From cf70ee0cf1312197f5b4abc18d9d099a63e070d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 3 Mar 2025 21:26:28 +0000 Subject: [PATCH 02/58] Add tests for replacing old value after nested property null merges --- tests/unit/onyxTest.ts | 272 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index 4ec0b7163..93e16ff8a 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -1730,6 +1730,157 @@ describe('Onyx', () => { }); }); + it('should replace the old value after a null merge in a nested property when batching updates', async () => { + let result: unknown; + connection = Onyx.connect({ + key: ONYX_KEYS.COLLECTION.TEST_UPDATE, + waitForCollectionCallback: true, + callback: (value) => { + result = value; + }, + }); + + await Onyx.multiSet({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + sub_entry1: { + id: 'sub_entry1', + someKey: 'someValue', + }, + }, + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { + sub_entry2: { + id: 'sub_entry2', + someKey: 'someValue', + someNestedObject: { + someNestedKey: 'someNestedValue', + anotherNestedObject: { + anotherNestedKey: 'anotherNestedValue', + }, + }, + }, + }, + }); + + const queuedUpdates: OnyxUpdate[] = [ + // entry1 + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + value: { + // Removing "sub_entry1" property in this update. + // Any subsequent changes to this property should completely replace the old value. + sub_entry1: null, + }, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + value: { + // This change should completely replace "sub_entry1" old value. + sub_entry1: { + newKey: 'newValue', + }, + }, + }, + + // entry2 + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + onyxMethod: 'merge', + value: { + sub_entry2: { + someKey: 'someValueChanged', + }, + }, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + onyxMethod: 'merge', + value: { + sub_entry2: { + someNestedObject: { + anotherNestedObject: { + anotherNestedKey: 'anotherNestedValueChanged', + }, + anotherNestedObject2: { + anotherNestedKey2: 'anotherNestedValue2', + }, + }, + }, + }, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + onyxMethod: 'merge', + value: { + sub_entry2: { + someNestedObject: { + anotherNestedObject: { + // Removing "anotherNestedKey" property in this update. + anotherNestedKey: null, + }, + }, + }, + }, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + onyxMethod: 'merge', + value: { + sub_entry2: { + someNestedObject: { + anotherNestedObject: { + newNestedKey: 'newNestedValue', + }, + // Removing "anotherNestedObject2" property in this update. + // Any subsequent changes to this property should completely replace the old value. + anotherNestedObject2: null, + }, + }, + }, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + onyxMethod: 'merge', + value: { + sub_entry2: { + someNestedObject: { + // This change should completely replace "anotherNestedObject2" old value. + anotherNestedObject2: { + newNestedKey2: 'newNestedValue2', + }, + }, + }, + }, + }, + ]; + + await Onyx.update(queuedUpdates); + + expect(result).toEqual({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + sub_entry1: { + newKey: 'newValue', + }, + }, + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { + sub_entry2: { + id: 'sub_entry2', + someKey: 'someValueChanged', + someNestedObject: { + someNestedKey: 'someNestedValue', + anotherNestedObject: { + newNestedKey: 'newNestedValue', + }, + anotherNestedObject2: { + newNestedKey2: 'newNestedValue2', + }, + }, + }, + }, + }); + }); + describe('merge', () => { it('should replace the old value after a null merge in the top-level object when batching merges', async () => { let result: unknown; @@ -1766,6 +1917,127 @@ describe('Onyx', () => { }); }); + it('should replace the old value after a null merge in a nested property when batching merges', async () => { + let result: unknown; + connection = Onyx.connect({ + key: ONYX_KEYS.COLLECTION.TEST_UPDATE, + waitForCollectionCallback: true, + callback: (value) => { + result = value; + }, + }); + + await Onyx.multiSet({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + sub_entry1: { + id: 'sub_entry1', + someKey: 'someValue', + }, + }, + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { + sub_entry2: { + id: 'sub_entry2', + someKey: 'someValue', + someNestedObject: { + someNestedKey: 'someNestedValue', + anotherNestedObject: { + anotherNestedKey: 'anotherNestedValue', + }, + }, + }, + }, + }); + + // entry1 + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + // Removing "sub_entry1" property in this merge. + // Any subsequent changes to this property should completely replace the old value. + sub_entry1: null, + }); + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + // This change should completely replace "sub_entry1" old value. + sub_entry1: { + newKey: 'newValue', + }, + }); + + // entry2 + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { + sub_entry2: { + someKey: 'someValueChanged', + }, + }); + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { + sub_entry2: { + someNestedObject: { + anotherNestedObject: { + anotherNestedKey: 'anotherNestedValueChanged', + }, + anotherNestedObject2: { + anotherNestedKey2: 'anotherNestedValue2', + }, + }, + }, + }); + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { + sub_entry2: { + someNestedObject: { + anotherNestedObject: { + // Removing "anotherNestedKey" property in this merge. + anotherNestedKey: null, + }, + }, + }, + }); + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { + sub_entry2: { + someNestedObject: { + anotherNestedObject: { + newNestedKey: 'newNestedValue', + }, + // Removing "anotherNestedObject2" property in this update. + // Any subsequent changes to this property should completely replace the old value. + anotherNestedObject2: null, + }, + }, + }); + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { + sub_entry2: { + someNestedObject: { + // This change should completely replace "anotherNestedObject2" old value. + anotherNestedObject2: { + newNestedKey2: 'newNestedValue2', + }, + }, + }, + }); + + await waitForPromisesToResolve(); + + expect(result).toEqual({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + sub_entry1: { + newKey: 'newValue', + }, + }, + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { + sub_entry2: { + id: 'sub_entry2', + someKey: 'someValueChanged', + someNestedObject: { + someNestedKey: 'someNestedValue', + anotherNestedObject: { + newNestedKey: 'newNestedValue', + }, + anotherNestedObject2: { + newNestedKey2: 'newNestedValue2', + }, + }, + }, + }, + }); + }); + it('should remove a deeply nested null when merging an existing key', () => { let result: unknown; From 2eca1c6286de2572c607abb39bd1345cbe56865c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 6 Mar 2025 18:58:58 +0000 Subject: [PATCH 03/58] Fix tests --- tests/unit/onyxTest.ts | 48 +++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index 93e16ff8a..47f6c491b 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -1756,6 +1756,9 @@ describe('Onyx', () => { anotherNestedObject: { anotherNestedKey: 'anotherNestedValue', }, + anotherNestedObject2: { + anotherNestedKey2: 'anotherNestedValue2', + }, }, }, }, @@ -1802,8 +1805,8 @@ describe('Onyx', () => { anotherNestedObject: { anotherNestedKey: 'anotherNestedValueChanged', }, - anotherNestedObject2: { - anotherNestedKey2: 'anotherNestedValue2', + anotherNestedObject3: { + anotherNestedKey3: 'anotherNestedValue3', }, }, }, @@ -1817,8 +1820,12 @@ describe('Onyx', () => { someNestedObject: { anotherNestedObject: { // Removing "anotherNestedKey" property in this update. + // This property's old value is a primitive value, so it won't cause issues with merging. anotherNestedKey: null, }, + // Removing "anotherNestedObject2" property in this update. + // Any subsequent changes to this property should completely replace the old value. + anotherNestedObject2: null, }, }, }, @@ -1832,9 +1839,10 @@ describe('Onyx', () => { anotherNestedObject: { newNestedKey: 'newNestedValue', }, - // Removing "anotherNestedObject2" property in this update. - // Any subsequent changes to this property should completely replace the old value. - anotherNestedObject2: null, + // Removing "anotherNestedObject3" property in this update. + // This property was only introduced in a previous update, so we don't need to care + // about an old value because there isn't one. + anotherNestedObject3: null, }, }, }, @@ -1849,6 +1857,9 @@ describe('Onyx', () => { anotherNestedObject2: { newNestedKey2: 'newNestedValue2', }, + anotherNestedObject3: { + newNestedKey3: 'newNestedValue3', + }, }, }, }, @@ -1875,6 +1886,9 @@ describe('Onyx', () => { anotherNestedObject2: { newNestedKey2: 'newNestedValue2', }, + anotherNestedObject3: { + newNestedKey3: 'newNestedValue3', + }, }, }, }, @@ -1943,6 +1957,9 @@ describe('Onyx', () => { anotherNestedObject: { anotherNestedKey: 'anotherNestedValue', }, + anotherNestedObject2: { + anotherNestedKey2: 'anotherNestedValue2', + }, }, }, }, @@ -1973,8 +1990,8 @@ describe('Onyx', () => { anotherNestedObject: { anotherNestedKey: 'anotherNestedValueChanged', }, - anotherNestedObject2: { - anotherNestedKey2: 'anotherNestedValue2', + anotherNestedObject3: { + anotherNestedKey3: 'anotherNestedValue3', }, }, }, @@ -1984,8 +2001,12 @@ describe('Onyx', () => { someNestedObject: { anotherNestedObject: { // Removing "anotherNestedKey" property in this merge. + // This property's old value is a primitive value, so it won't cause issues with merging. anotherNestedKey: null, }, + // Removing "anotherNestedObject2" property in this merge. + // Any subsequent changes to this property should completely replace the old value. + anotherNestedObject2: null, }, }, }); @@ -1995,9 +2016,10 @@ describe('Onyx', () => { anotherNestedObject: { newNestedKey: 'newNestedValue', }, - // Removing "anotherNestedObject2" property in this update. - // Any subsequent changes to this property should completely replace the old value. - anotherNestedObject2: null, + // Removing "anotherNestedObject3" property in this merge. + // This property was only introduced in a previous merge, so we don't need to care + // about an old value because there isn't one. + anotherNestedObject3: null, }, }, }); @@ -2008,6 +2030,9 @@ describe('Onyx', () => { anotherNestedObject2: { newNestedKey2: 'newNestedValue2', }, + anotherNestedObject3: { + newNestedKey3: 'newNestedValue3', + }, }, }, }); @@ -2032,6 +2057,9 @@ describe('Onyx', () => { anotherNestedObject2: { newNestedKey2: 'newNestedValue2', }, + anotherNestedObject3: { + newNestedKey3: 'newNestedValue3', + }, }, }, }, From 4e61d186baf86758b63e1933bafc589c5f6ebd64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 10 Mar 2025 18:11:34 +0000 Subject: [PATCH 04/58] Use a temporary marker during merging to replace old values when applicable --- lib/Onyx.ts | 8 ++++---- lib/OnyxUtils.ts | 7 ++++++- lib/utils.ts | 26 ++++++++++++++++++++++---- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index a02dc75f7..ca01ce112 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -320,7 +320,7 @@ function merge(key: TKey, changes: OnyxMergeInput): if (!validChanges.length) { return Promise.resolve(); } - const batchedDeltaChanges = OnyxUtils.applyMerge(undefined, validChanges, false); + const batchedDeltaChanges = OnyxUtils.applyMerge(undefined, validChanges, false, true, false); // Case (1): When there is no existing value in storage, we want to set the value instead of merge it. // Case (2): The presence of a top-level `null` in the merge queue instructs us to drop the whole existing value. @@ -350,7 +350,7 @@ function merge(key: TKey, changes: OnyxMergeInput): // The "preMergedValue" will be directly "set" in storage instead of being merged // Therefore we merge the batched changes with the existing value to get the final merged value that will be stored. // We can remove null values from the "preMergedValue", because "null" implicates that the user wants to remove a value from storage. - const preMergedValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges], true); + const preMergedValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges], true, false, true); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. const hasChanged = cache.hasValueChanged(key, preMergedValue); @@ -770,7 +770,7 @@ function update(data: OnyxUpdate[]): Promise { // Remove the collection-related key from the updateQueue so that it won't be processed individually. delete updateQueue[key]; - const updatedValue = OnyxUtils.applyMerge(undefined, operations, false); + const updatedValue = OnyxUtils.applyMerge(undefined, operations, false, true, false); if (operations[0] === null) { // eslint-disable-next-line no-param-reassign queue.set[key] = updatedValue; @@ -795,7 +795,7 @@ function update(data: OnyxUpdate[]): Promise { }); Object.entries(updateQueue).forEach(([key, operations]) => { - const batchedChanges = OnyxUtils.applyMerge(undefined, operations, false); + const batchedChanges = OnyxUtils.applyMerge(undefined, operations, false, true, false); if (operations[0] === null) { promises.push(() => set(key, batchedChanges)); diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index de1c4eada..ee0bf17ae 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1259,6 +1259,8 @@ function applyMerge | undefined, TChange exten existingValue: TValue, changes: TChange[], shouldRemoveNestedNulls: boolean, + isBatchingMergeChanges: boolean, + shouldReplaceMarkedObjects: boolean, ): TChange { const lastChange = changes?.at(-1); @@ -1268,7 +1270,10 @@ function applyMerge | undefined, TChange exten if (changes.some((change) => change && typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNestedNulls), (existingValue || {}) as TChange); + return changes.reduce( + (modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects), + (existingValue || {}) as TChange, + ); } // If we have anything else we can't merge it so we'll diff --git a/lib/utils.ts b/lib/utils.ts index cdd04a54e..87aa27fbc 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -5,6 +5,8 @@ import type {ConnectOptions, OnyxInput, OnyxKey} from './types'; type EmptyObject = Record; type EmptyValue = EmptyObject | null | undefined; +const ONYX_INTERNALS__REPLACE_OBJECT_MARK = 'ONYX_INTERNALS__REPLACE_OBJECT_MARK'; + /** Checks whether the given object is an object and not null/undefined. */ function isEmptyObject(obj: T | EmptyValue): obj is EmptyValue { return typeof obj === 'object' && Object.keys(obj || {}).length === 0; @@ -27,7 +29,13 @@ function isMergeableObject(value: unknown): value is Record { * @param shouldRemoveNestedNulls - If true, null object values will be removed. * @returns - The merged object. */ -function mergeObject>(target: TObject | unknown | null | undefined, source: TObject, shouldRemoveNestedNulls = true): TObject { +function mergeObject>( + target: TObject | unknown | null | undefined, + source: TObject, + shouldRemoveNestedNulls = true, + isBatchingMergeChanges = false, + shouldReplaceMarkedObjects = false, +): TObject { const destination: Record = {}; const targetObject = isMergeableObject(target) ? target : undefined; @@ -78,7 +86,17 @@ function mergeObject>(target: TObject | // so that we can still use "fastMerge" to merge the source value, // to ensure that nested null values are removed from the merged object. const targetValueWithFallback = (targetValue ?? {}) as TObject; - destination[key] = fastMerge(targetValueWithFallback, sourceValue, shouldRemoveNestedNulls); + + if (isBatchingMergeChanges && targetValue === null) { + (targetValueWithFallback as Record)[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; + } + + if (shouldReplaceMarkedObjects && sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK] === true) { + delete sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK]; + destination[key] = sourceValue; + } else { + destination[key] = fastMerge(targetValueWithFallback, sourceValue, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects); + } } else { destination[key] = sourceValue; } @@ -95,7 +113,7 @@ function mergeObject>(target: TObject | * On native, when merging an existing value with new changes, SQLite will use JSON_PATCH, which removes top-level nullish values. * To be consistent with the behaviour for merge, we'll also want to remove null values for "set" operations. */ -function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNulls = true): TValue { +function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNulls = true, isBatchingMergeChanges = false, shouldReplaceMarkedObjects = false): TValue { // We have to ignore arrays and nullish values here, // otherwise "mergeObject" will throw an error, // because it expects an object as "source" @@ -103,7 +121,7 @@ function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNul return source; } - return mergeObject(target, source as Record, shouldRemoveNestedNulls) as TValue; + return mergeObject(target, source as Record, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects) as TValue; } /** Deep removes the nested null values from the given value. */ From 404af80ac1f050172fafac3ab8a98ce7a3c1b72b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 24 Mar 2025 16:05:14 +0000 Subject: [PATCH 05/58] Change mergeItem() to not use delta changes anymore --- lib/Onyx.ts | 2 +- lib/storage/index.ts | 4 ++-- lib/storage/providers/IDBKeyValProvider.ts | 2 +- lib/storage/providers/MemoryOnlyProvider.ts | 2 +- lib/storage/providers/SQLiteProvider.ts | 8 ++------ lib/storage/providers/types.ts | 4 +--- 6 files changed, 8 insertions(+), 14 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index ca01ce112..dd4beb7f8 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -365,7 +365,7 @@ function merge(key: TKey, changes: OnyxMergeInput): return updatePromise; } - return Storage.mergeItem(key, batchedDeltaChanges as OnyxValue, preMergedValue as OnyxValue, shouldSetValue).then(() => { + return Storage.mergeItem(key, preMergedValue as OnyxValue).then(() => { OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, preMergedValue); return updatePromise; }); diff --git a/lib/storage/index.ts b/lib/storage/index.ts index 938b615a5..f2d2d33a7 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -116,9 +116,9 @@ const storage: Storage = { /** * Merging an existing value with a new one */ - mergeItem: (key, deltaChanges, preMergedValue, shouldSetValue = false) => + mergeItem: (key, preMergedValue) => tryOrDegradePerformance(() => { - const promise = provider.mergeItem(key, deltaChanges, preMergedValue, shouldSetValue); + const promise = provider.mergeItem(key, preMergedValue); if (shouldKeepInstancesSync) { return promise.then(() => InstanceSync.mergeItem(key)); diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index 9b749ab86..92700daf1 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -55,7 +55,7 @@ const provider: StorageProvider = { return Promise.all(upsertMany); }); }), - mergeItem(key, _deltaChanges, preMergedValue) { + mergeItem(key, preMergedValue) { // Since Onyx also merged the existing value with the changes, we can just set the value directly return provider.setItem(key, preMergedValue); }, diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index f67cff12a..7ca025153 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -74,7 +74,7 @@ const provider: StorageProvider = { /** * Merging an existing value with a new one */ - mergeItem(key, _deltaChanges, preMergedValue) { + mergeItem(key, preMergedValue) { // Since Onyx already merged the existing value with the changes, we can just set the value directly return this.setItem(key, preMergedValue); }, diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 461c4b8ca..1201c491b 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -76,12 +76,8 @@ const provider: StorageProvider = { return db.executeBatchAsync([[query, queryArguments]]); }, - mergeItem(key, deltaChanges, preMergedValue, shouldSetValue) { - if (shouldSetValue) { - return this.setItem(key, preMergedValue) as Promise; - } - - return this.multiMerge([[key, deltaChanges]]) as Promise; + mergeItem(key, preMergedValue) { + return this.setItem(key, preMergedValue) as Promise; }, getAllKeys: () => db.executeAsync('SELECT record_key FROM keyvaluepairs;').then(({rows}) => { diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index 6552602af..c4ee1316e 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -43,11 +43,9 @@ type StorageProvider = { /** * Merges an existing value with a new one by leveraging JSON_PATCH - * @param deltaChanges - the delta for a specific key * @param preMergedValue - the pre-merged data from `Onyx.applyMerge` - * @param shouldSetValue - whether the data should be set instead of merged */ - mergeItem: (key: TKey, deltaChanges: OnyxValue, preMergedValue: OnyxValue, shouldSetValue?: boolean) => Promise; + mergeItem: (key: TKey, preMergedValue: OnyxValue) => Promise; /** * Returns all keys available in storage From 2d1ceb1e1df8592bf2ac999c575543d0c52fd6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 24 Mar 2025 16:07:13 +0000 Subject: [PATCH 06/58] Change multiMerge() to not use JSON_PATCH anymore --- lib/OnyxCache.ts | 2 +- lib/OnyxUtils.ts | 2 +- lib/storage/providers/IDBKeyValProvider.ts | 2 +- lib/storage/providers/MemoryOnlyProvider.ts | 2 +- lib/storage/providers/SQLiteProvider.ts | 22 +++++++++------------ lib/utils.ts | 14 ++++++------- tests/perf-test/utils.perf-test.ts | 2 +- tests/unit/fastMergeTest.ts | 10 +++++----- 8 files changed, 26 insertions(+), 30 deletions(-) diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index a6df22f7e..7227881ef 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -164,7 +164,7 @@ class OnyxCache { throw new Error('data passed to cache.merge() must be an Object of onyx key/value pairs'); } - this.storageMap = {...utils.fastMerge(this.storageMap, data)}; + this.storageMap = {...utils.fastMerge(this.storageMap, data, true, false, true)}; Object.entries(data).forEach(([key, value]) => { this.addKey(key); diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index ee0bf17ae..335013798 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1288,7 +1288,7 @@ function initializeWithDefaultKeyStates(): Promise { return Storage.multiGet(Object.keys(defaultKeyStates)).then((pairs) => { const existingDataAsObject = Object.fromEntries(pairs); - const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates); + const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, true, false, false); cache.merge(merged ?? {}); Object.entries(merged ?? {}).forEach(([key, value]) => keyChanged(key, value, existingDataAsObject)); diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index 92700daf1..ecd70e041 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -49,7 +49,7 @@ const provider: StorageProvider = { const upsertMany = pairsWithoutNull.map(([key, value], index) => { const prev = values[index]; - const newValue = utils.fastMerge(prev as Record, value as Record); + const newValue = utils.fastMerge(prev as Record, value as Record, true, false, true); return promisifyRequest(store.put(newValue, key)); }); return Promise.all(upsertMany); diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index 7ca025153..5070d1f65 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -86,7 +86,7 @@ const provider: StorageProvider = { multiMerge(pairs) { _.forEach(pairs, ([key, value]) => { const existingValue = store[key] as Record; - const newValue = utils.fastMerge(existingValue, value as Record) as OnyxValue; + const newValue = utils.fastMerge(existingValue, value as Record, true, false, true) as OnyxValue; set(key, newValue); }); diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 1201c491b..6f5228b51 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -60,21 +60,17 @@ const provider: StorageProvider = { return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]); }, multiMerge(pairs) { - // Note: We use `ON CONFLICT DO UPDATE` here instead of `INSERT OR REPLACE INTO` - // so the new JSON value is merged into the old one if there's an existing value - const query = `INSERT INTO keyvaluepairs (record_key, valueJSON) - VALUES (:key, JSON(:value)) - ON CONFLICT DO UPDATE - SET valueJSON = JSON_PATCH(valueJSON, JSON(:value)); - `; - const nonNullishPairs = pairs.filter((pair) => pair[1] !== undefined); - const queryArguments = nonNullishPairs.map((pair) => { - const value = JSON.stringify(pair[1]); - return [pair[0], value]; - }); + const nonNullishPairsKeys = pairs.map((pair) => pair[0]); - return db.executeBatchAsync([[query, queryArguments]]); + return this.multiGet(nonNullishPairsKeys).then((storagePairs) => { + const newPairs: KeyValuePairList = []; + for (let i = 0; i < storagePairs.length; i++) { + const newPair = storagePairs[i]; + newPairs[i] = [newPair[0], utils.fastMerge(newPair[1] as Record, nonNullishPairs[i][1] as Record, true, false, true)]; + } + return this.multiSet(newPairs); + }); }, mergeItem(key, preMergedValue) { return this.setItem(key, preMergedValue) as Promise; diff --git a/lib/utils.ts b/lib/utils.ts index 87aa27fbc..6f01bc3d1 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -32,9 +32,9 @@ function isMergeableObject(value: unknown): value is Record { function mergeObject>( target: TObject | unknown | null | undefined, source: TObject, - shouldRemoveNestedNulls = true, - isBatchingMergeChanges = false, - shouldReplaceMarkedObjects = false, + shouldRemoveNestedNulls: boolean, + isBatchingMergeChanges: boolean, + shouldReplaceMarkedObjects: boolean, ): TObject { const destination: Record = {}; @@ -92,8 +92,8 @@ function mergeObject>( } if (shouldReplaceMarkedObjects && sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK] === true) { - delete sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK]; - destination[key] = sourceValue; + destination[key] = {...sourceValue}; + delete (destination[key] as Record).ONYX_INTERNALS__REPLACE_OBJECT_MARK; } else { destination[key] = fastMerge(targetValueWithFallback, sourceValue, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects); } @@ -113,7 +113,7 @@ function mergeObject>( * On native, when merging an existing value with new changes, SQLite will use JSON_PATCH, which removes top-level nullish values. * To be consistent with the behaviour for merge, we'll also want to remove null values for "set" operations. */ -function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNulls = true, isBatchingMergeChanges = false, shouldReplaceMarkedObjects = false): TValue { +function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNulls: boolean, isBatchingMergeChanges: boolean, shouldReplaceMarkedObjects: boolean): TValue { // We have to ignore arrays and nullish values here, // otherwise "mergeObject" will throw an error, // because it expects an object as "source" @@ -128,7 +128,7 @@ function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNul function removeNestedNullValues | null>(value: TValue): TValue { if (typeof value === 'object' && !Array.isArray(value)) { const objectValue = value as Record; - return fastMerge(objectValue, objectValue) as TValue; + return fastMerge(objectValue, objectValue, true, false, true) as TValue; } return value; diff --git a/tests/perf-test/utils.perf-test.ts b/tests/perf-test/utils.perf-test.ts index f87f44a6d..b3ab893c7 100644 --- a/tests/perf-test/utils.perf-test.ts +++ b/tests/perf-test/utils.perf-test.ts @@ -15,6 +15,6 @@ describe('[Utils.js]', () => { const target = getMockedPersonalDetails(1000); const source = getMockedPersonalDetails(500); - await measureFunction(() => utils.fastMerge(target, source, true)); + await measureFunction(() => utils.fastMerge(target, source, true, false, false)); }); }); diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index d20ba80ea..f49333038 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -38,7 +38,7 @@ const testObjectWithNullValuesRemoved: DeepObject = { describe('fastMerge', () => { it('should merge an object with another object and remove nested null values', () => { - const result = utils.fastMerge(testObject, testObjectWithNullishValues); + const result = utils.fastMerge(testObject, testObjectWithNullishValues, true, false, false); expect(result).toEqual({ a: 'a', @@ -55,7 +55,7 @@ describe('fastMerge', () => { }); it('should merge an object with another object and not remove nested null values', () => { - const result = utils.fastMerge(testObject, testObjectWithNullishValues, false); + const result = utils.fastMerge(testObject, testObjectWithNullishValues, false, false, false); expect(result).toEqual({ a: 'a', @@ -73,7 +73,7 @@ describe('fastMerge', () => { }); it('should merge an object with an empty object and remove deeply nested null values', () => { - const result = utils.fastMerge({}, testObjectWithNullishValues); + const result = utils.fastMerge({}, testObjectWithNullishValues, true, false, false); expect(result).toEqual(testObjectWithNullValuesRemoved); }); @@ -86,14 +86,14 @@ describe('fastMerge', () => { it('should replace an object with an array', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = utils.fastMerge(testObject, [1, 2, 3] as any); + const result = utils.fastMerge(testObject, [1, 2, 3] as any, true, false, false); expect(result).toEqual([1, 2, 3]); }); it('should replace an array with an object', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = utils.fastMerge([1, 2, 3] as any, testObject); + const result = utils.fastMerge([1, 2, 3] as any, testObject, true, false, false); expect(result).toEqual(testObject); }); From a619757e18cc701e5512826077f58f6fd58d0f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Sat, 29 Mar 2025 18:06:30 +0000 Subject: [PATCH 07/58] Add comments --- lib/Onyx.ts | 27 ++++++++++++--------- lib/storage/providers/IDBKeyValProvider.ts | 2 +- lib/storage/providers/MemoryOnlyProvider.ts | 2 +- lib/storage/providers/SQLiteProvider.ts | 1 + lib/storage/providers/types.ts | 2 +- lib/utils.ts | 17 +++++++++++-- 6 files changed, 34 insertions(+), 17 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index dd4beb7f8..5516c6c48 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -307,8 +307,10 @@ function merge(key: TKey, changes: OnyxMergeInput): } try { - // We first only merge the changes, so we can provide these to the native implementation (SQLite uses only delta changes in "JSON_PATCH" to merge) - // We don't want to remove null values from the "batchedDeltaChanges", because SQLite uses them to remove keys from storage natively. + // We first only merge the changes, so we can use our custom merging strategy by signalling OnyxUtils.applyMerge() + // that we are batching merge changes. + // We don't want to remove null values from the "batchedDeltaChanges" at the moment, this process will be done when merging + // the batched changes to the existing value. const validChanges = mergeQueue[key].filter((change) => { const {isCompatible, existingValueType, newValueType} = utils.checkCompatibilityWithExistingValue(change, existingValue); if (!isCompatible) { @@ -324,19 +326,19 @@ function merge(key: TKey, changes: OnyxMergeInput): // Case (1): When there is no existing value in storage, we want to set the value instead of merge it. // Case (2): The presence of a top-level `null` in the merge queue instructs us to drop the whole existing value. - // In this case, we can't simply merge the batched changes with the existing value, because then the null in the merge queue would have no effect + // In this case, we can't simply merge the batched changes with the existing value, because then the null in the merge queue would have no effect. const shouldSetValue = !existingValue || mergeQueue[key].includes(null); - // Clean up the write queue, so we don't apply these changes again + // Clean up the write queue, so we don't apply these changes again. delete mergeQueue[key]; delete mergeQueuePromise[key]; const logMergeCall = (hasChanged = true) => { - // Logging properties only since values could be sensitive things we don't want to log + // Logging properties only since values could be sensitive things we don't want to log. Logger.logInfo(`merge called for key: ${key}${_.isObject(batchedDeltaChanges) ? ` properties: ${_.keys(batchedDeltaChanges).join(',')}` : ''} hasChanged: ${hasChanged}`); }; - // If the batched changes equal null, we want to remove the key from storage, to reduce storage size + // If the batched changes equal null, we want to remove the key from storage, to reduce storage size. const {wasRemoved} = OnyxUtils.removeNullValues(key, batchedDeltaChanges); // Calling "OnyxUtils.removeNullValues" removes the key from storage and cache and updates the subscriber. @@ -346,10 +348,11 @@ function merge(key: TKey, changes: OnyxMergeInput): return Promise.resolve(); } - // For providers that can't handle delta changes, we need to merge the batched changes with the existing value beforehand. - // The "preMergedValue" will be directly "set" in storage instead of being merged - // Therefore we merge the batched changes with the existing value to get the final merged value that will be stored. + // The "preMergedValue" will be directly "set" in storage instead of being merged, so we merge + // the batched changes with the existing value to get the final merged value that will be stored. // We can remove null values from the "preMergedValue", because "null" implicates that the user wants to remove a value from storage. + // Additionally, we will signal OnyxUtils.applyMerge() to replace any nested properties previously marked in "batchedDeltaChanges" + // by our custom merging strategy. const preMergedValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges], true, false, true); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. @@ -770,13 +773,13 @@ function update(data: OnyxUpdate[]): Promise { // Remove the collection-related key from the updateQueue so that it won't be processed individually. delete updateQueue[key]; - const updatedValue = OnyxUtils.applyMerge(undefined, operations, false, true, false); + const batchedChanges = OnyxUtils.applyMerge(undefined, operations, false, true, false); if (operations[0] === null) { // eslint-disable-next-line no-param-reassign - queue.set[key] = updatedValue; + queue.set[key] = batchedChanges; } else { // eslint-disable-next-line no-param-reassign - queue.merge[key] = updatedValue; + queue.merge[key] = batchedChanges; } return queue; }, diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index ecd70e041..b2afd8b16 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -56,7 +56,7 @@ const provider: StorageProvider = { }); }), mergeItem(key, preMergedValue) { - // Since Onyx also merged the existing value with the changes, we can just set the value directly + // Since Onyx already merged the existing value with the changes, we can just set the value directly. return provider.setItem(key, preMergedValue); }, multiSet: (pairs) => { diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index 5070d1f65..7c784b75e 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -75,7 +75,7 @@ const provider: StorageProvider = { * Merging an existing value with a new one */ mergeItem(key, preMergedValue) { - // Since Onyx already merged the existing value with the changes, we can just set the value directly + // Since Onyx already merged the existing value with the changes, we can just set the value directly. return this.setItem(key, preMergedValue); }, diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 6f5228b51..78d665297 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -73,6 +73,7 @@ const provider: StorageProvider = { }); }, mergeItem(key, preMergedValue) { + // Since Onyx already merged the existing value with the changes, we can just set the value directly. return this.setItem(key, preMergedValue) as Promise; }, getAllKeys: () => diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index c4ee1316e..c189cf377 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -42,7 +42,7 @@ type StorageProvider = { multiMerge: (pairs: KeyValuePairList) => Promise; /** - * Merges an existing value with a new one by leveraging JSON_PATCH + * Merges an existing value with a new one * @param preMergedValue - the pre-merged data from `Onyx.applyMerge` */ mergeItem: (key: TKey, preMergedValue: OnyxValue) => Promise; diff --git a/lib/utils.ts b/lib/utils.ts index 6f01bc3d1..3a20c323c 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -27,6 +27,10 @@ function isMergeableObject(value: unknown): value is Record { * @param target - The target object. * @param source - The source object. * @param shouldRemoveNestedNulls - If true, null object values will be removed. + * @param isBatchingMergeChanges - If true, it means that we are batching merge changes before applying + * them to the Onyx value, so we must use a special logic to handle these changes. + * @param shouldReplaceMarkedObjects - If true, any nested objects that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" + * flag will be completely replaced instead of merged. * @returns - The merged object. */ function mergeObject>( @@ -87,14 +91,25 @@ function mergeObject>( // to ensure that nested null values are removed from the merged object. const targetValueWithFallback = (targetValue ?? {}) as TObject; + // If we are batching merge changes and the previous merge change (targetValue) is null, + // it means we want to fully replace this object when merging the batched changes with the Onyx value. + // To achieve this, we first mark these nested objects with an internal flag. With the desired objects + // marked, when calling this method again with "shouldReplaceMarkedObjects" set to true we can proceed + // to effectively replace them in the next condition. if (isBatchingMergeChanges && targetValue === null) { (targetValueWithFallback as Record)[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; } + // Then, when merging the batched changes with the Onyx value, if a nested object of the batched changes + // has the internal flag set, we replace the entire destination object with the source one and remove + // the flag. if (shouldReplaceMarkedObjects && sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK] === true) { + // We do a spread here in order to have a new object reference and allow us to delete the internal flag + // of the merged object only. destination[key] = {...sourceValue}; delete (destination[key] as Record).ONYX_INTERNALS__REPLACE_OBJECT_MARK; } else { + // For the normal situations we'll just call `fastMerge()` again to merge the nested object. destination[key] = fastMerge(targetValueWithFallback, sourceValue, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects); } } else { @@ -110,8 +125,6 @@ function mergeObject>( * Merges two objects and removes null values if "shouldRemoveNestedNulls" is set to true * * We generally want to remove null values from objects written to disk and cache, because it decreases the amount of data stored in memory and on disk. - * On native, when merging an existing value with new changes, SQLite will use JSON_PATCH, which removes top-level nullish values. - * To be consistent with the behaviour for merge, we'll also want to remove null values for "set" operations. */ function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNulls: boolean, isBatchingMergeChanges: boolean, shouldReplaceMarkedObjects: boolean): TValue { // We have to ignore arrays and nullish values here, From aa28a4d01cb02d17fb5a7d44141311c0b5ff8f5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Sun, 30 Mar 2025 18:43:01 +0100 Subject: [PATCH 08/58] Add tests to applyMerge() --- lib/utils.ts | 12 +++- tests/unit/onyxUtilsTest.ts | 121 ++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/lib/utils.ts b/lib/utils.ts index 3a20c323c..d89a2749c 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -234,4 +234,14 @@ function hasWithOnyxInstance(mapping: ConnectOptions return 'withOnyxInstance' in mapping && mapping.withOnyxInstance; } -export default {isEmptyObject, fastMerge, formatActionName, removeNestedNullValues, checkCompatibilityWithExistingValue, pick, omit, hasWithOnyxInstance}; +export default { + isEmptyObject, + fastMerge, + formatActionName, + removeNestedNullValues, + checkCompatibilityWithExistingValue, + pick, + omit, + hasWithOnyxInstance, + ONYX_INTERNALS__REPLACE_OBJECT_MARK, +}; diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index 06e6be1d1..23aa6cfc4 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -1,5 +1,7 @@ import Onyx from '../../lib'; import OnyxUtils from '../../lib/OnyxUtils'; +import type {DeepRecord} from '../../lib/types'; +import utils from '../../lib/utils'; const ONYXKEYS = { TEST_KEY: 'test', @@ -87,4 +89,123 @@ describe('OnyxUtils', () => { }).toThrowError("Invalid '' key provided, only collection keys are allowed."); }); }); + + describe('applyMerge', () => { + const testObject: DeepRecord = { + a: 'a', + b: { + c: 'c', + d: { + e: 'e', + f: 'f', + }, + g: 'g', + }, + }; + + const testMergeChanges: Array> = [ + { + b: { + d: { + h: 'h', + }, + }, + }, + { + b: { + d: null, + h: 'h', + }, + }, + { + b: { + d: { + i: 'i', + }, + }, + }, + { + b: { + d: null, + g: null, + }, + }, + { + b: { + d: { + i: 'i', + j: 'j', + }, + g: { + k: 'k', + }, + }, + }, + ]; + + it("should return the last change if it's an array", () => { + const result = OnyxUtils.applyMerge(testObject, [...testMergeChanges, [0, 1, 2]], false, false, true); + + expect(result).toEqual([0, 1, 2]); + }); + + it("should return the last change if the changes aren't objects", () => { + const result = OnyxUtils.applyMerge(testObject, ['a', 0, 'b', 1], false, false, true); + + expect(result).toEqual(1); + }); + + it('should merge data correctly when batching merge changes', () => { + const result = OnyxUtils.applyMerge(undefined, testMergeChanges, false, true, false); + + expect(result).toEqual({ + b: { + d: { + i: 'i', + j: 'j', + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + }, + h: 'h', + g: { + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + k: 'k', + }, + }, + }); + }); + + it('should merge data correctly when applying batched changes', () => { + const batchedChanges: DeepRecord = { + b: { + d: { + i: 'i', + j: 'j', + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + }, + h: 'h', + g: { + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + k: 'k', + }, + }, + }; + + const result = OnyxUtils.applyMerge(testObject, [batchedChanges], false, false, true); + + expect(result).toEqual({ + a: 'a', + b: { + c: 'c', + d: { + i: 'i', + j: 'j', + }, + h: 'h', + g: { + k: 'k', + }, + }, + }); + }); + }); }); From 934e768f263cf1aa1a6eef12c3cc063f0146a53b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Sun, 30 Mar 2025 19:46:26 +0100 Subject: [PATCH 09/58] Improve and fix native multiMerge logic --- lib/storage/providers/SQLiteProvider.ts | 41 ++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 78d665297..79e6f2f38 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -8,6 +8,7 @@ import {getFreeDiskStorage} from 'react-native-device-info'; import type StorageProvider from './types'; import utils from '../../utils'; import type {KeyList, KeyValuePairList} from './types'; +import type {OnyxKey, OnyxValue} from '../../types'; const DB_NAME = 'OnyxDB'; let db: QuickSQLiteConnection; @@ -60,15 +61,45 @@ const provider: StorageProvider = { return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]); }, multiMerge(pairs) { - const nonNullishPairs = pairs.filter((pair) => pair[1] !== undefined); - const nonNullishPairsKeys = pairs.map((pair) => pair[0]); + const nonNullishPairs: KeyValuePairList = []; + const nonNullishPairsKeys: OnyxKey[] = []; + + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < pairs.length; i++) { + const pair = pairs[i]; + if (pair[1] !== undefined) { + nonNullishPairs.push(pair); + nonNullishPairsKeys.push(pair[0]); + } + } + + if (nonNullishPairs.length === 0) { + return Promise.resolve(); + } return this.multiGet(nonNullishPairsKeys).then((storagePairs) => { - const newPairs: KeyValuePairList = []; + // multiGet() is not guaranteed to return the data in the same order we asked with "nonNullishPairsKeys", + // so we use a map to associate keys to their existing values correctly. + const existingMap = new Map>(); + // eslint-disable-next-line @typescript-eslint/prefer-for-of for (let i = 0; i < storagePairs.length; i++) { - const newPair = storagePairs[i]; - newPairs[i] = [newPair[0], utils.fastMerge(newPair[1] as Record, nonNullishPairs[i][1] as Record, true, false, true)]; + existingMap.set(storagePairs[i][0], storagePairs[i][1]); } + + const newPairs: KeyValuePairList = []; + + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < nonNullishPairs.length; i++) { + const key = nonNullishPairs[i][0]; + const newValue = nonNullishPairs[i][1]; + + const existingValue = existingMap.get(key) ?? {}; + + const mergedValue = utils.fastMerge(existingValue, newValue, true, false, true); + + newPairs.push([key, mergedValue]); + } + return this.multiSet(newPairs); }); }, From 1b8c8042be4a275a46b1ba985fa872b969fa1444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Sun, 6 Apr 2025 18:14:41 +0100 Subject: [PATCH 10/58] Address small review comments --- lib/Onyx.ts | 2 +- lib/utils.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 5516c6c48..58761abfb 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -307,7 +307,7 @@ function merge(key: TKey, changes: OnyxMergeInput): } try { - // We first only merge the changes, so we can use our custom merging strategy by signalling OnyxUtils.applyMerge() + // We first only merge the changes, so we can use our custom merging strategy by signaling OnyxUtils.applyMerge() // that we are batching merge changes. // We don't want to remove null values from the "batchedDeltaChanges" at the moment, this process will be done when merging // the batched changes to the existing value. diff --git a/lib/utils.ts b/lib/utils.ts index d89a2749c..566c5797c 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -95,7 +95,7 @@ function mergeObject>( // it means we want to fully replace this object when merging the batched changes with the Onyx value. // To achieve this, we first mark these nested objects with an internal flag. With the desired objects // marked, when calling this method again with "shouldReplaceMarkedObjects" set to true we can proceed - // to effectively replace them in the next condition. + // effectively replace them in the next condition. if (isBatchingMergeChanges && targetValue === null) { (targetValueWithFallback as Record)[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; } @@ -103,7 +103,7 @@ function mergeObject>( // Then, when merging the batched changes with the Onyx value, if a nested object of the batched changes // has the internal flag set, we replace the entire destination object with the source one and remove // the flag. - if (shouldReplaceMarkedObjects && sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK] === true) { + if (shouldReplaceMarkedObjects && sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { // We do a spread here in order to have a new object reference and allow us to delete the internal flag // of the merged object only. destination[key] = {...sourceValue}; From 19e2d2030a85368e15d812e1260b0d5bd88a8818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 7 Apr 2025 07:32:11 +0100 Subject: [PATCH 11/58] Add tests for fastMerge --- tests/unit/fastMergeTest.ts | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index f49333038..01614f076 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -36,6 +36,22 @@ const testObjectWithNullValuesRemoved: DeepObject = { }, }; +const testMergeChanges: DeepObject[] = [ + { + b: { + d: { + h: 'h', + }, + }, + }, + { + b: { + d: null, + h: 'h', + }, + }, +]; + describe('fastMerge', () => { it('should merge an object with another object and remove nested null values', () => { const result = utils.fastMerge(testObject, testObjectWithNullishValues, true, false, false); @@ -97,4 +113,48 @@ describe('fastMerge', () => { expect(result).toEqual(testObject); }); + + it('should add the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag to the target object when its source is set to null and "isBatchingMergeChanges" is true', () => { + const result = utils.fastMerge(testMergeChanges[1], testMergeChanges[0], true, true, false); + + expect(result).toEqual({ + b: { + d: { + h: 'h', + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + }, + h: 'h', + }, + }); + }); + + it('should completely replace the target object with its source when the source has the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag and "shouldReplaceMarkedObjects" is true', () => { + const result = utils.fastMerge( + testObject, + { + b: { + d: { + h: 'h', + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + }, + h: 'h', + }, + }, + true, + false, + true, + ); + + expect(result).toEqual({ + a: 'a', + b: { + c: 'c', + d: { + h: 'h', + }, + h: 'h', + g: 'g', + }, + }); + }); }); From befaaeb378285522b42212d09dba4d731520c372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Sun, 13 Apr 2025 17:48:22 +0100 Subject: [PATCH 12/58] Improve tests --- tests/unit/onyxTest.ts | 454 ++++++++++++++++++++++------------------- 1 file changed, 247 insertions(+), 207 deletions(-) diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index 0f94e841a..a25a58af6 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -1,9 +1,10 @@ import lodashClone from 'lodash/clone'; +import lodashCloneDeep from 'lodash/cloneDeep'; import Onyx from '../../lib'; import waitForPromisesToResolve from '../utils/waitForPromisesToResolve'; import OnyxUtils from '../../lib/OnyxUtils'; import type OnyxCache from '../../lib/OnyxCache'; -import type {OnyxCollection, OnyxUpdate} from '../../lib/types'; +import type {DeepRecord, OnyxCollection, OnyxUpdate} from '../../lib/types'; import type GenericCollection from '../utils/GenericCollection'; import type {Connection} from '../../lib/OnyxConnectionManager'; @@ -1742,168 +1743,190 @@ describe('Onyx', () => { }); }); - it('should replace the old value after a null merge in a nested property when batching updates', async () => { + describe('should replace the old value after a null merge in a nested property when batching updates', () => { let result: unknown; - connection = Onyx.connect({ - key: ONYX_KEYS.COLLECTION.TEST_UPDATE, - waitForCollectionCallback: true, - callback: (value) => { - result = value; - }, + + beforeEach(() => { + connection = Onyx.connect({ + key: ONYX_KEYS.COLLECTION.TEST_UPDATE, + waitForCollectionCallback: true, + callback: (value) => { + result = value; + }, + }); }); - await Onyx.multiSet({ - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + it('replacing old object after null merge', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', }, - }, - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { - sub_entry2: { - id: 'sub_entry2', - someKey: 'someValue', - someNestedObject: { - someNestedKey: 'someNestedValue', - anotherNestedObject: { - anotherNestedKey: 'anotherNestedValue', - }, - anotherNestedObject2: { - anotherNestedKey2: 'anotherNestedValue2', - }, - }, - }, - }, - }); + }; + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1}); - const queuedUpdates: OnyxUpdate[] = [ - // entry1 - { + const entry1ExpectedResult = lodashCloneDeep(entry1); + const queuedUpdates: OnyxUpdate[] = []; + + queuedUpdates.push({ key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, onyxMethod: 'merge', value: { - // Removing "sub_entry1" property in this update. - // Any subsequent changes to this property should completely replace the old value. + // Removing the "sub_entry1" object in this update. + // Any subsequent changes to this object should completely replace the existing object in store. sub_entry1: null, }, - }, - { + }); + delete entry1ExpectedResult.sub_entry1; + + queuedUpdates.push({ key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, onyxMethod: 'merge', value: { - // This change should completely replace "sub_entry1" old value. + // This change should completely replace "sub_entry1" existing object in store. sub_entry1: { newKey: 'newValue', }, }, - }, + }); + entry1ExpectedResult.sub_entry1 = {newKey: 'newValue'}; - // entry2 - { - key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, - onyxMethod: 'merge', - value: { - sub_entry2: { - someKey: 'someValueChanged', - }, - }, - }, - { - key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, - onyxMethod: 'merge', - value: { - sub_entry2: { - someNestedObject: { - anotherNestedObject: { - anotherNestedKey: 'anotherNestedValueChanged', - }, - anotherNestedObject3: { - anotherNestedKey3: 'anotherNestedValue3', - }, + await Onyx.update(queuedUpdates); + + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + }); + + it('setting new object after null merge', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord = { + sub_entry1: { + id: 'sub_entry1', + someKey: 'someValue', + someNestedObject: { + someNestedKey: 'someNestedValue', + anotherNestedObject: { + anotherNestedKey: 'anotherNestedValue', }, }, }, - }, - { - key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + }; + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1}); + + const entry1ExpectedResult = lodashCloneDeep(entry1); + const queuedUpdates: OnyxUpdate[] = []; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, onyxMethod: 'merge', value: { - sub_entry2: { + sub_entry1: { someNestedObject: { - anotherNestedObject: { - // Removing "anotherNestedKey" property in this update. - // This property's old value is a primitive value, so it won't cause issues with merging. - anotherNestedKey: null, + // Introducing a new "anotherNestedObject2" object in this update. + anotherNestedObject2: { + anotherNestedKey2: 'anotherNestedValue2', }, - // Removing "anotherNestedObject2" property in this update. - // Any subsequent changes to this property should completely replace the old value. - anotherNestedObject2: null, }, }, }, - }, - { - key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + }); + entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject2 = {anotherNestedKey2: 'anotherNestedValue2'}; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, onyxMethod: 'merge', value: { - sub_entry2: { + sub_entry1: { someNestedObject: { - anotherNestedObject: { - newNestedKey: 'newNestedValue', - }, - // Removing "anotherNestedObject3" property in this update. + // Removing the "anotherNestedObject2" object in this update. // This property was only introduced in a previous update, so we don't need to care - // about an old value because there isn't one. - anotherNestedObject3: null, + // about an old existing value because there isn't one. + anotherNestedObject2: null, }, }, }, - }, - { - key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + }); + delete entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject2; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, onyxMethod: 'merge', value: { - sub_entry2: { + sub_entry1: { someNestedObject: { - // This change should completely replace "anotherNestedObject2" old value. + // Introducing the "anotherNestedObject2" object again with this update. anotherNestedObject2: { newNestedKey2: 'newNestedValue2', }, - anotherNestedObject3: { - newNestedKey3: 'newNestedValue3', - }, }, }, }, - }, - ]; + }); + entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject2 = {newNestedKey2: 'newNestedValue2'}; - await Onyx.update(queuedUpdates); + await Onyx.update(queuedUpdates); - expect(result).toEqual({ - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + }); + + it('setting new object after null merge of a primitive property', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord = { sub_entry1: { - newKey: 'newValue', - }, - }, - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { - sub_entry2: { - id: 'sub_entry2', - someKey: 'someValueChanged', + id: 'sub_entry1', + someKey: 'someValue', someNestedObject: { someNestedKey: 'someNestedValue', anotherNestedObject: { - newNestedKey: 'newNestedValue', + anotherNestedKey: 'anotherNestedValue', }, - anotherNestedObject2: { - newNestedKey2: 'newNestedValue2', + }, + }, + }; + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1}); + + const entry1ExpectedResult = lodashCloneDeep(entry1); + const queuedUpdates: OnyxUpdate[] = []; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + value: { + sub_entry1: { + someNestedObject: { + anotherNestedObject: { + // Removing the "anotherNestedKey" property in this update. + // This property's existing value in store is a primitive value, so we don't need to care + // about it when merging new values in any next updates. + anotherNestedKey: null, + }, }, - anotherNestedObject3: { - newNestedKey3: 'newNestedValue3', + }, + }, + }); + delete entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject.anotherNestedKey; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + value: { + sub_entry1: { + someNestedObject: { + anotherNestedObject: { + // Setting a new object to the "anotherNestedKey" property. + anotherNestedKey: { + newNestedKey: 'newNestedValue', + }, + }, }, }, }, - }, + }); + entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject.anotherNestedKey = {newNestedKey: 'newNestedValue'}; + + await Onyx.update(queuedUpdates); + + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); }); }); @@ -1936,145 +1959,162 @@ describe('Onyx', () => { await waitForPromisesToResolve(); - expect(result).toEqual({ - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { - someKey: 'someValueChanged', - }, - }); + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: {someKey: 'someValueChanged'}}); }); - it('should replace the old value after a null merge in a nested property when batching merges', async () => { + describe('should replace the old value after a null merge in a nested property when batching merges', () => { let result: unknown; - connection = Onyx.connect({ - key: ONYX_KEYS.COLLECTION.TEST_UPDATE, - waitForCollectionCallback: true, - callback: (value) => { - result = value; - }, + + beforeEach(() => { + connection = Onyx.connect({ + key: ONYX_KEYS.COLLECTION.TEST_UPDATE, + waitForCollectionCallback: true, + callback: (value) => { + result = value; + }, + }); }); - await Onyx.multiSet({ - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + it('replacing old object after null merge', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', }, - }, - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { - sub_entry2: { - id: 'sub_entry2', + }; + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1}); + + const entry1ExpectedResult = lodashCloneDeep(entry1); + + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + // Removing the "sub_entry1" object in this merge. + // Any subsequent changes to this object should completely replace the existing object in store. + sub_entry1: null, + }); + delete entry1ExpectedResult.sub_entry1; + + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + // This change should completely replace "sub_entry1" existing object in store. + sub_entry1: { + newKey: 'newValue', + }, + }); + entry1ExpectedResult.sub_entry1 = {newKey: 'newValue'}; + + await waitForPromisesToResolve(); + + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + }); + + it('setting new object after null merge', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord = { + sub_entry1: { + id: 'sub_entry1', someKey: 'someValue', someNestedObject: { someNestedKey: 'someNestedValue', anotherNestedObject: { anotherNestedKey: 'anotherNestedValue', }, - anotherNestedObject2: { - anotherNestedKey2: 'anotherNestedValue2', - }, }, }, - }, - }); + }; + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1}); - // entry1 - Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { - // Removing "sub_entry1" property in this merge. - // Any subsequent changes to this property should completely replace the old value. - sub_entry1: null, - }); - Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { - // This change should completely replace "sub_entry1" old value. - sub_entry1: { - newKey: 'newValue', - }, - }); + const entry1ExpectedResult = lodashCloneDeep(entry1); - // entry2 - Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { - sub_entry2: { - someKey: 'someValueChanged', - }, - }); - Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { - sub_entry2: { - someNestedObject: { - anotherNestedObject: { - anotherNestedKey: 'anotherNestedValueChanged', - }, - anotherNestedObject3: { - anotherNestedKey3: 'anotherNestedValue3', + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + sub_entry1: { + someNestedObject: { + // Introducing a new "anotherNestedObject2" object in this merge. + anotherNestedObject2: { + anotherNestedKey2: 'anotherNestedValue2', + }, }, }, - }, - }); - Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { - sub_entry2: { - someNestedObject: { - anotherNestedObject: { - // Removing "anotherNestedKey" property in this merge. - // This property's old value is a primitive value, so it won't cause issues with merging. - anotherNestedKey: null, + }); + entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject2 = {anotherNestedKey2: 'anotherNestedValue2'}; + + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + sub_entry1: { + someNestedObject: { + // Removing the "anotherNestedObject2" object in this merge. + // This property was only introduced in a previous merge, so we don't need to care + // about an old existing value because there isn't one. + anotherNestedObject2: null, }, - // Removing "anotherNestedObject2" property in this merge. - // Any subsequent changes to this property should completely replace the old value. - anotherNestedObject2: null, }, - }, - }); - Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { - sub_entry2: { - someNestedObject: { - anotherNestedObject: { - newNestedKey: 'newNestedValue', + }); + delete entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject2; + + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + sub_entry1: { + someNestedObject: { + // Introducing the "anotherNestedObject2" object again with this update. + anotherNestedObject2: { + newNestedKey2: 'newNestedValue2', + }, }, - // Removing "anotherNestedObject3" property in this merge. - // This property was only introduced in a previous merge, so we don't need to care - // about an old value because there isn't one. - anotherNestedObject3: null, }, - }, + }); + entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject2 = {newNestedKey2: 'newNestedValue2'}; + + await waitForPromisesToResolve(); + + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); }); - Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, { - sub_entry2: { - someNestedObject: { - // This change should completely replace "anotherNestedObject2" old value. - anotherNestedObject2: { - newNestedKey2: 'newNestedValue2', - }, - anotherNestedObject3: { - newNestedKey3: 'newNestedValue3', + + it('setting new object after null merge of a primitive property', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord = { + sub_entry1: { + id: 'sub_entry1', + someKey: 'someValue', + someNestedObject: { + someNestedKey: 'someNestedValue', + anotherNestedObject: { + anotherNestedKey: 'anotherNestedValue', + }, }, }, - }, - }); + }; + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1}); - await waitForPromisesToResolve(); + const entry1ExpectedResult = lodashCloneDeep(entry1); - expect(result).toEqual({ - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { sub_entry1: { - newKey: 'newValue', - }, - }, - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: { - sub_entry2: { - id: 'sub_entry2', - someKey: 'someValueChanged', someNestedObject: { - someNestedKey: 'someNestedValue', anotherNestedObject: { - newNestedKey: 'newNestedValue', - }, - anotherNestedObject2: { - newNestedKey2: 'newNestedValue2', + // Removing the "anotherNestedKey" property in this merge. + // This property's existing value in store is a primitive value, so we don't need to care + // about it when merging new values in any next merges. + anotherNestedKey: null, }, - anotherNestedObject3: { - newNestedKey3: 'newNestedValue3', + }, + }, + }); + delete entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject.anotherNestedKey; + + Onyx.merge(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, { + sub_entry1: { + someNestedObject: { + anotherNestedObject: { + // Setting a new object to the "anotherNestedKey" property. + anotherNestedKey: { + newNestedKey: 'newNestedValue', + }, }, }, }, - }, + }); + entry1ExpectedResult.sub_entry1.someNestedObject.anotherNestedObject.anotherNestedKey = {newNestedKey: 'newNestedValue'}; + + await waitForPromisesToResolve(); + + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); }); }); From df65c33c9d7583ca2a9742350b7aa66456dca729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Sun, 13 Apr 2025 18:37:43 +0100 Subject: [PATCH 13/58] Simplify and extract applyMerge() batching merges logic to a separate batchMergeChanges() function --- lib/Onyx.ts | 13 ++-- lib/OnyxUtils.ts | 31 +++++--- tests/unit/onyxUtilsTest.ts | 147 +++++++++++++++++++----------------- 3 files changed, 104 insertions(+), 87 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 58761abfb..394554fc9 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -307,10 +307,7 @@ function merge(key: TKey, changes: OnyxMergeInput): } try { - // We first only merge the changes, so we can use our custom merging strategy by signaling OnyxUtils.applyMerge() - // that we are batching merge changes. - // We don't want to remove null values from the "batchedDeltaChanges" at the moment, this process will be done when merging - // the batched changes to the existing value. + // We first only merge the changes, so we use OnyxUtils.batchMergeChanges() to combine all the changes into just one. const validChanges = mergeQueue[key].filter((change) => { const {isCompatible, existingValueType, newValueType} = utils.checkCompatibilityWithExistingValue(change, existingValue); if (!isCompatible) { @@ -322,7 +319,7 @@ function merge(key: TKey, changes: OnyxMergeInput): if (!validChanges.length) { return Promise.resolve(); } - const batchedDeltaChanges = OnyxUtils.applyMerge(undefined, validChanges, false, true, false); + const batchedDeltaChanges = OnyxUtils.batchMergeChanges(validChanges); // Case (1): When there is no existing value in storage, we want to set the value instead of merge it. // Case (2): The presence of a top-level `null` in the merge queue instructs us to drop the whole existing value. @@ -353,7 +350,7 @@ function merge(key: TKey, changes: OnyxMergeInput): // We can remove null values from the "preMergedValue", because "null" implicates that the user wants to remove a value from storage. // Additionally, we will signal OnyxUtils.applyMerge() to replace any nested properties previously marked in "batchedDeltaChanges" // by our custom merging strategy. - const preMergedValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges], true, false, true); + const preMergedValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges]); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. const hasChanged = cache.hasValueChanged(key, preMergedValue); @@ -773,7 +770,7 @@ function update(data: OnyxUpdate[]): Promise { // Remove the collection-related key from the updateQueue so that it won't be processed individually. delete updateQueue[key]; - const batchedChanges = OnyxUtils.applyMerge(undefined, operations, false, true, false); + const batchedChanges = OnyxUtils.batchMergeChanges(operations); if (operations[0] === null) { // eslint-disable-next-line no-param-reassign queue.set[key] = batchedChanges; @@ -798,7 +795,7 @@ function update(data: OnyxUpdate[]): Promise { }); Object.entries(updateQueue).forEach(([key, operations]) => { - const batchedChanges = OnyxUtils.applyMerge(undefined, operations, false, true, false); + const batchedChanges = OnyxUtils.batchMergeChanges(operations); if (operations[0] === null) { promises.push(() => set(key, batchedChanges)); diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 0b200d394..0069e7156 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1255,13 +1255,7 @@ function prepareKeyValuePairsForStorage(data: Record * * @param changes Array of changes that should be applied to the existing value */ -function applyMerge | undefined, TChange extends OnyxInput | undefined>( - existingValue: TValue, - changes: TChange[], - shouldRemoveNestedNulls: boolean, - isBatchingMergeChanges: boolean, - shouldReplaceMarkedObjects: boolean, -): TChange { +function applyMerge | undefined, TChange extends OnyxInput | undefined>(existingValue: TValue, changes: TChange[]): TChange { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { @@ -1270,10 +1264,24 @@ function applyMerge | undefined, TChange exten if (changes.some((change) => change && typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce( - (modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects), - (existingValue || {}) as TChange, - ); + return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, true, false, true), (existingValue || {}) as TChange); + } + + // If we have anything else we can't merge it so we'll + // simply return the last value that was queued + return lastChange as TChange; +} + +function batchMergeChanges | undefined>(changes: TChange[]): TChange { + const lastChange = changes?.at(-1); + + if (Array.isArray(lastChange)) { + return lastChange; + } + + if (changes.some((change) => change && typeof change === 'object')) { + // Object values are then merged one after the other + return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, false, true, false), {} as TChange); } // If we have anything else we can't merge it so we'll @@ -1490,6 +1498,7 @@ const OnyxUtils = { getEvictionBlocklist, getSkippableCollectionMemberIDs, setSkippableCollectionMemberIDs, + batchMergeChanges, }; GlobalSettings.addGlobalSettingsChangeListener(({enablePerformanceMetrics}) => { diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index 23aa6cfc4..7479ced74 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -3,6 +3,67 @@ import OnyxUtils from '../../lib/OnyxUtils'; import type {DeepRecord} from '../../lib/types'; import utils from '../../lib/utils'; +const testObject: DeepRecord = { + a: 'a', + b: { + c: 'c', + d: { + e: 'e', + f: 'f', + }, + g: 'g', + }, +}; + +const testMergeChanges: Array> = [ + { + b: { + d: { + h: 'h', + }, + }, + }, + { + b: { + // Removing "d" object. + d: null, + h: 'h', + }, + }, + { + b: { + // Adding back "d" property with a object. + // The "ONYX_INTERNALS__REPLACE_OBJECT_MARK" marker property should be added here when batching merge changes. + d: { + i: 'i', + }, + }, + }, + { + b: { + // Removing "d" object again. + d: null, + // Removing "g" object. + g: null, + }, + }, + { + b: { + // Adding back "d" property with a object. + // The "ONYX_INTERNALS__REPLACE_OBJECT_MARK" marker property should be added here when batching merge changes. + d: { + i: 'i', + j: 'j', + }, + // Adding back "g" property with a object. + // The "ONYX_INTERNALS__REPLACE_OBJECT_MARK" marker property should be added here when batching merge changes. + g: { + k: 'k', + }, + }, + }, +]; + const ONYXKEYS = { TEST_KEY: 'test', COLLECTION: { @@ -91,74 +152,20 @@ describe('OnyxUtils', () => { }); describe('applyMerge', () => { - const testObject: DeepRecord = { - a: 'a', - b: { - c: 'c', - d: { - e: 'e', - f: 'f', - }, - g: 'g', - }, - }; - - const testMergeChanges: Array> = [ - { - b: { - d: { - h: 'h', - }, - }, - }, - { - b: { - d: null, - h: 'h', - }, - }, - { - b: { - d: { - i: 'i', - }, - }, - }, - { - b: { - d: null, - g: null, - }, - }, - { - b: { - d: { - i: 'i', - j: 'j', - }, - g: { - k: 'k', - }, - }, - }, - ]; - it("should return the last change if it's an array", () => { - const result = OnyxUtils.applyMerge(testObject, [...testMergeChanges, [0, 1, 2]], false, false, true); + const result = OnyxUtils.applyMerge(testObject, [...testMergeChanges, [0, 1, 2]]); expect(result).toEqual([0, 1, 2]); }); it("should return the last change if the changes aren't objects", () => { - const result = OnyxUtils.applyMerge(testObject, ['a', 0, 'b', 1], false, false, true); + const result = OnyxUtils.applyMerge(testObject, ['a', 0, 'b', 1]); expect(result).toEqual(1); }); - it('should merge data correctly when batching merge changes', () => { - const result = OnyxUtils.applyMerge(undefined, testMergeChanges, false, true, false); - - expect(result).toEqual({ + it('should merge data correctly when applying batched changes', () => { + const batchedChanges: DeepRecord = { b: { d: { i: 'i', @@ -171,37 +178,41 @@ describe('OnyxUtils', () => { k: 'k', }, }, - }); - }); + }; - it('should merge data correctly when applying batched changes', () => { - const batchedChanges: DeepRecord = { + const result = OnyxUtils.applyMerge(testObject, [batchedChanges]); + + expect(result).toEqual({ + a: 'a', b: { + c: 'c', d: { i: 'i', j: 'j', - [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, }, h: 'h', g: { - [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, k: 'k', }, }, - }; + }); + }); + }); - const result = OnyxUtils.applyMerge(testObject, [batchedChanges], false, false, true); + describe('batchMergeChanges', () => { + it('should apply the replacement markers if the we have properties with objects being removed and added back during the changes', () => { + const result = OnyxUtils.batchMergeChanges(testMergeChanges); expect(result).toEqual({ - a: 'a', b: { - c: 'c', d: { i: 'i', j: 'j', + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, }, h: 'h', g: { + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, k: 'k', }, }, From 6301dfdae441d0605ff4fe798cf4f69ebc9be241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Sun, 13 Apr 2025 19:01:15 +0100 Subject: [PATCH 14/58] Improve comments in Onyx.merge() --- lib/Onyx.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 394554fc9..daeffb41b 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -345,28 +345,28 @@ function merge(key: TKey, changes: OnyxMergeInput): return Promise.resolve(); } - // The "preMergedValue" will be directly "set" in storage instead of being merged, so we merge - // the batched changes with the existing value to get the final merged value that will be stored. - // We can remove null values from the "preMergedValue", because "null" implicates that the user wants to remove a value from storage. - // Additionally, we will signal OnyxUtils.applyMerge() to replace any nested properties previously marked in "batchedDeltaChanges" - // by our custom merging strategy. - const preMergedValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges]); + // If "shouldSetValue" is true, it means that we want to completely replace the existing value with the batched changes, + // so we pass `undefined` to OnyxUtils.applyMerge() first parameter to make it use "batchedDeltaChanges" to + // create a new object for us. + // If "shouldSetValue" is false, it means that we want to merge the batched changes into the existing value, + // so we pass "existingValue" to the first parameter. + const resultValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges]); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. - const hasChanged = cache.hasValueChanged(key, preMergedValue); + const hasChanged = cache.hasValueChanged(key, resultValue); logMergeCall(hasChanged); // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. - const updatePromise = OnyxUtils.broadcastUpdate(key, preMergedValue as OnyxValue, hasChanged); + const updatePromise = OnyxUtils.broadcastUpdate(key, resultValue as OnyxValue, hasChanged); // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. if (!hasChanged) { return updatePromise; } - return Storage.mergeItem(key, preMergedValue as OnyxValue).then(() => { - OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, preMergedValue); + return Storage.mergeItem(key, resultValue as OnyxValue).then(() => { + OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, resultValue); return updatePromise; }); } catch (error) { From bd928cdf6df9ac39559d10794086a0a2c68b8ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 12 May 2025 10:40:56 +0100 Subject: [PATCH 15/58] First experiments with JSON_PATCH + JSON_REPLACE solution --- lib/storage/providers/SQLiteProvider.ts | 111 ++++++++++++++++-------- 1 file changed, 76 insertions(+), 35 deletions(-) diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 79e6f2f38..8d9b1f000 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -2,7 +2,7 @@ * The SQLiteStorage provider stores everything in a key/value store by * converting the value to a JSON string */ -import type {BatchQueryResult, QuickSQLiteConnection} from 'react-native-quick-sqlite'; +import type {BatchQueryResult, QuickSQLiteConnection, SQLBatchTuple} from 'react-native-quick-sqlite'; import {open} from 'react-native-quick-sqlite'; import {getFreeDiskStorage} from 'react-native-device-info'; import type StorageProvider from './types'; @@ -13,6 +13,54 @@ import type {OnyxKey, OnyxValue} from '../../types'; const DB_NAME = 'OnyxDB'; let db: QuickSQLiteConnection; +function replacer(key: string, value: unknown) { + if (key === utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK) return undefined; + return value; +} + +type JSONReplacePatch = [string, string[], any]; + +function getReplacePatches(storageKey: string, value: any): JSONReplacePatch[] { + const patches: JSONReplacePatch[] = []; + + // eslint-disable-next-line rulesdir/prefer-early-return + function recurse(obj: any, path: string[] = []) { + if (obj && typeof obj === 'object' && !Array.isArray(obj)) { + if (obj.ONYX_INTERNALS__REPLACE_OBJECT_MARK) { + const copy = {...obj}; + delete copy.ONYX_INTERNALS__REPLACE_OBJECT_MARK; + + patches.push([storageKey, [...path], copy]); + return; + } + + // eslint-disable-next-line guard-for-in, no-restricted-syntax + for (const key in obj) { + recurse(obj[key], [...path, key]); + } + } + } + + recurse(value); + return patches; +} + +function generateJSONReplaceSQLBatch(patches: JSONReplacePatch[]): [string, string[][]] { + const sql = ` + UPDATE keyvaluepairs + SET valueJSON = JSON_REPLACE(valueJSON, :jsonPath, JSON(:value)) + WHERE record_key = :key; + `; + + const queryArguments = patches.map(([key, pathArray, value]) => { + const jsonPath = `$.${pathArray.join('.')}`; + // return {key, jsonPath, value: JSON.stringify(value)}; + return [jsonPath, JSON.stringify(value), key]; + }); + + return [sql.trim(), queryArguments]; +} + const provider: StorageProvider = { /** * The name of the provider that can be printed to the logs @@ -61,47 +109,40 @@ const provider: StorageProvider = { return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]); }, multiMerge(pairs) { - const nonNullishPairs: KeyValuePairList = []; - const nonNullishPairsKeys: OnyxKey[] = []; + const commands: SQLBatchTuple[] = []; + const patchQuery = `INSERT INTO keyvaluepairs (record_key, valueJSON) + VALUES (:key, JSON(:value)) + ON CONFLICT DO UPDATE + SET valueJSON = JSON_PATCH(valueJSON, JSON(:value)); + `; + const patchQueryArguments: string[][] = []; + const replaceQuery = `UPDATE keyvaluepairs + SET valueJSON = JSON_REPLACE(valueJSON, ?, JSON(?)) + WHERE record_key = ?; + `; + const replaceQueryArguments: string[][] = []; + + const nonNullishPairs = pairs.filter((pair) => pair[1] !== undefined); // eslint-disable-next-line @typescript-eslint/prefer-for-of - for (let i = 0; i < pairs.length; i++) { - const pair = pairs[i]; - if (pair[1] !== undefined) { - nonNullishPairs.push(pair); - nonNullishPairsKeys.push(pair[0]); + for (let i = 0; i < nonNullishPairs.length; i++) { + const pair = nonNullishPairs[i]; + const value = JSON.stringify(pair[1], replacer); + patchQueryArguments.push([pair[0], value]); + + const patches = getReplacePatches(pair[0], pair[1]); + const [sql, args] = generateJSONReplaceSQLBatch(patches); + if (args.length > 0) { + replaceQueryArguments.push(...args); } } - if (nonNullishPairs.length === 0) { - return Promise.resolve(); + commands.push([patchQuery, patchQueryArguments]); + if (replaceQueryArguments.length > 0) { + commands.push([replaceQuery, replaceQueryArguments]); } - return this.multiGet(nonNullishPairsKeys).then((storagePairs) => { - // multiGet() is not guaranteed to return the data in the same order we asked with "nonNullishPairsKeys", - // so we use a map to associate keys to their existing values correctly. - const existingMap = new Map>(); - // eslint-disable-next-line @typescript-eslint/prefer-for-of - for (let i = 0; i < storagePairs.length; i++) { - existingMap.set(storagePairs[i][0], storagePairs[i][1]); - } - - const newPairs: KeyValuePairList = []; - - // eslint-disable-next-line @typescript-eslint/prefer-for-of - for (let i = 0; i < nonNullishPairs.length; i++) { - const key = nonNullishPairs[i][0]; - const newValue = nonNullishPairs[i][1]; - - const existingValue = existingMap.get(key) ?? {}; - - const mergedValue = utils.fastMerge(existingValue, newValue, true, false, true); - - newPairs.push([key, mergedValue]); - } - - return this.multiSet(newPairs); - }); + return db.executeBatchAsync(commands); }, mergeItem(key, preMergedValue) { // Since Onyx already merged the existing value with the changes, we can just set the value directly. From d88c9dd61dd5d5aef9c07cea506dd68056a853c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 15 May 2025 08:26:13 +0100 Subject: [PATCH 16/58] Refactor and move getReplacePatches() logic to fastMerge() / mergeObject() --- lib/Onyx.ts | 25 ++++++--- lib/OnyxCache.ts | 2 +- lib/OnyxUtils.ts | 26 ++++++--- lib/storage/index.ts | 4 +- lib/storage/providers/IDBKeyValProvider.ts | 2 +- lib/storage/providers/MemoryOnlyProvider.ts | 2 +- lib/storage/providers/SQLiteProvider.ts | 60 ++++++--------------- lib/storage/providers/types.ts | 4 +- lib/types.ts | 2 + lib/utils.ts | 46 +++++++++++++--- tests/unit/fastMergeTest.ts | 15 +++--- tests/unit/onyxUtilsTest.ts | 57 +++++++++++++++++++- 12 files changed, 166 insertions(+), 79 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 012162132..c89d24f38 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -319,7 +319,7 @@ function merge(key: TKey, changes: OnyxMergeInput): if (!validChanges.length) { return Promise.resolve(); } - const batchedDeltaChanges = OnyxUtils.batchMergeChanges(validChanges); + const batchedDeltaChanges = OnyxUtils.batchMergeChanges(validChanges).result; // Case (1): When there is no existing value in storage, we want to set the value instead of merge it. // Case (2): The presence of a top-level `null` in the merge queue instructs us to drop the whole existing value. @@ -391,7 +391,11 @@ function merge(key: TKey, changes: OnyxMergeInput): * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` * @param collection Object collection keyed by individual collection member keys and values */ -function mergeCollection(collectionKey: TKey, collection: OnyxMergeCollectionInput): Promise { +function mergeCollection( + collectionKey: TKey, + collection: OnyxMergeCollectionInput, + mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches'], +): Promise { if (!OnyxUtils.isValidNonEmptyCollectionForMerge(collection)) { Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.'); return Promise.resolve(); @@ -476,7 +480,7 @@ function mergeCollection(collectionKey: TK // New keys will be added via multiSet while existing keys will be updated using multiMerge // This is because setting a key that doesn't exist yet with multiMerge will throw errors if (keyValuePairsForExistingCollection.length > 0) { - promises.push(Storage.multiMerge(keyValuePairsForExistingCollection)); + promises.push(Storage.multiMerge(keyValuePairsForExistingCollection, mergeReplaceNullPatches)); } if (keyValuePairsForNewCollection.length > 0) { @@ -773,21 +777,28 @@ function update(data: OnyxUpdate[]): Promise { const batchedChanges = OnyxUtils.batchMergeChanges(operations); if (operations[0] === null) { // eslint-disable-next-line no-param-reassign - queue.set[key] = batchedChanges; + queue.set[key] = batchedChanges.result; } else { // eslint-disable-next-line no-param-reassign - queue.merge[key] = batchedChanges; + queue.merge[key] = batchedChanges.result; + if (batchedChanges.replaceNullPatches.length > 0) { + // eslint-disable-next-line no-param-reassign + queue.mergeReplaceNullPatches[key] = batchedChanges.replaceNullPatches; + } } return queue; }, { merge: {}, + mergeReplaceNullPatches: {}, set: {}, }, ); if (!utils.isEmptyObject(batchedCollectionUpdates.merge)) { - promises.push(() => mergeCollection(collectionKey, batchedCollectionUpdates.merge as Collection)); + promises.push(() => + mergeCollection(collectionKey, batchedCollectionUpdates.merge as Collection, batchedCollectionUpdates.mergeReplaceNullPatches), + ); } if (!utils.isEmptyObject(batchedCollectionUpdates.set)) { promises.push(() => multiSet(batchedCollectionUpdates.set)); @@ -795,7 +806,7 @@ function update(data: OnyxUpdate[]): Promise { }); Object.entries(updateQueue).forEach(([key, operations]) => { - const batchedChanges = OnyxUtils.batchMergeChanges(operations); + const batchedChanges = OnyxUtils.batchMergeChanges(operations).result; if (operations[0] === null) { promises.push(() => set(key, batchedChanges)); diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index 7227881ef..691cedfee 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -164,7 +164,7 @@ class OnyxCache { throw new Error('data passed to cache.merge() must be an Object of onyx key/value pairs'); } - this.storageMap = {...utils.fastMerge(this.storageMap, data, true, false, true)}; + this.storageMap = {...utils.fastMerge(this.storageMap, data, true, false, true).result}; Object.entries(data).forEach(([key, value]) => { this.addKey(key); diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 0069e7156..93b6bf651 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -28,6 +28,7 @@ import type { OnyxValue, Selector, } from './types'; +import type {FastMergeResult} from './utils'; import utils from './utils'; import type {WithOnyxState} from './withOnyx/types'; import type {DeferredTask} from './createDeferredTask'; @@ -1264,7 +1265,7 @@ function applyMerge | undefined, TChange exten if (changes.some((change) => change && typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, true, false, true), (existingValue || {}) as TChange); + return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, true, false, true).result, (existingValue || {}) as TChange); } // If we have anything else we can't merge it so we'll @@ -1272,21 +1273,34 @@ function applyMerge | undefined, TChange exten return lastChange as TChange; } -function batchMergeChanges | undefined>(changes: TChange[]): TChange { +function batchMergeChanges | undefined>(changes: TChange[]): FastMergeResult { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { - return lastChange; + return {result: lastChange, replaceNullPatches: []}; } if (changes.some((change) => change && typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, false, true, false), {} as TChange); + return changes.reduce>( + (modifiedData, change) => { + const fastMergeResult = utils.fastMerge(modifiedData.result, change, false, true, false); + // eslint-disable-next-line no-param-reassign + modifiedData.result = fastMergeResult.result; + // eslint-disable-next-line no-param-reassign + modifiedData.replaceNullPatches = [...modifiedData.replaceNullPatches, ...fastMergeResult.replaceNullPatches]; + return modifiedData; + }, + { + result: {} as TChange, + replaceNullPatches: [], + }, + ); } // If we have anything else we can't merge it so we'll // simply return the last value that was queued - return lastChange as TChange; + return {result: lastChange as TChange, replaceNullPatches: []}; } /** @@ -1296,7 +1310,7 @@ function initializeWithDefaultKeyStates(): Promise { return Storage.multiGet(Object.keys(defaultKeyStates)).then((pairs) => { const existingDataAsObject = Object.fromEntries(pairs); - const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, true, false, false); + const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, true, false, false).result; cache.merge(merged ?? {}); Object.entries(merged ?? {}).forEach(([key, value]) => keyChanged(key, value, existingDataAsObject)); diff --git a/lib/storage/index.ts b/lib/storage/index.ts index f2d2d33a7..33befed1d 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -131,9 +131,9 @@ const storage: Storage = { * Multiple merging of existing and new values in a batch * This function also removes all nested null values from an object. */ - multiMerge: (pairs) => + multiMerge: (pairs, mergeReplaceNullPatches) => tryOrDegradePerformance(() => { - const promise = provider.multiMerge(pairs); + const promise = provider.multiMerge(pairs, mergeReplaceNullPatches); if (shouldKeepInstancesSync) { return promise.then(() => InstanceSync.multiMerge(pairs.map((pair) => pair[0]))); diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index b2afd8b16..09fa6af5a 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -49,7 +49,7 @@ const provider: StorageProvider = { const upsertMany = pairsWithoutNull.map(([key, value], index) => { const prev = values[index]; - const newValue = utils.fastMerge(prev as Record, value as Record, true, false, true); + const newValue = utils.fastMerge(prev as Record, value as Record, true, false, true).result; return promisifyRequest(store.put(newValue, key)); }); return Promise.all(upsertMany); diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index 7c784b75e..7f1b1c2fb 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -86,7 +86,7 @@ const provider: StorageProvider = { multiMerge(pairs) { _.forEach(pairs, ([key, value]) => { const existingValue = store[key] as Record; - const newValue = utils.fastMerge(existingValue, value as Record, true, false, true) as OnyxValue; + const newValue = utils.fastMerge(existingValue, value as Record, true, false, true).result as OnyxValue; set(key, newValue); }); diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 8d9b1f000..d3dd7d7de 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -2,13 +2,13 @@ * The SQLiteStorage provider stores everything in a key/value store by * converting the value to a JSON string */ +import {getFreeDiskStorage} from 'react-native-device-info'; import type {BatchQueryResult, QuickSQLiteConnection, SQLBatchTuple} from 'react-native-quick-sqlite'; import {open} from 'react-native-quick-sqlite'; -import {getFreeDiskStorage} from 'react-native-device-info'; -import type StorageProvider from './types'; +import type {FastMergeReplaceNullPatch} from '../../utils'; import utils from '../../utils'; +import type StorageProvider from './types'; import type {KeyList, KeyValuePairList} from './types'; -import type {OnyxKey, OnyxValue} from '../../types'; const DB_NAME = 'OnyxDB'; let db: QuickSQLiteConnection; @@ -18,47 +18,13 @@ function replacer(key: string, value: unknown) { return value; } -type JSONReplacePatch = [string, string[], any]; - -function getReplacePatches(storageKey: string, value: any): JSONReplacePatch[] { - const patches: JSONReplacePatch[] = []; - - // eslint-disable-next-line rulesdir/prefer-early-return - function recurse(obj: any, path: string[] = []) { - if (obj && typeof obj === 'object' && !Array.isArray(obj)) { - if (obj.ONYX_INTERNALS__REPLACE_OBJECT_MARK) { - const copy = {...obj}; - delete copy.ONYX_INTERNALS__REPLACE_OBJECT_MARK; - - patches.push([storageKey, [...path], copy]); - return; - } - - // eslint-disable-next-line guard-for-in, no-restricted-syntax - for (const key in obj) { - recurse(obj[key], [...path, key]); - } - } - } - - recurse(value); - return patches; -} - -function generateJSONReplaceSQLBatch(patches: JSONReplacePatch[]): [string, string[][]] { - const sql = ` - UPDATE keyvaluepairs - SET valueJSON = JSON_REPLACE(valueJSON, :jsonPath, JSON(:value)) - WHERE record_key = :key; - `; - - const queryArguments = patches.map(([key, pathArray, value]) => { +function generateJSONReplaceSQLQueries(key: string, patches: FastMergeReplaceNullPatch[]): string[][] { + const queries = patches.map(([pathArray, value]) => { const jsonPath = `$.${pathArray.join('.')}`; - // return {key, jsonPath, value: JSON.stringify(value)}; return [jsonPath, JSON.stringify(value), key]; }); - return [sql.trim(), queryArguments]; + return queries; } const provider: StorageProvider = { @@ -108,7 +74,7 @@ const provider: StorageProvider = { } return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]); }, - multiMerge(pairs) { + multiMerge(pairs, mergeReplaceNullPatches) { const commands: SQLBatchTuple[] = []; const patchQuery = `INSERT INTO keyvaluepairs (record_key, valueJSON) @@ -124,16 +90,20 @@ const provider: StorageProvider = { const replaceQueryArguments: string[][] = []; const nonNullishPairs = pairs.filter((pair) => pair[1] !== undefined); + // eslint-disable-next-line @typescript-eslint/prefer-for-of for (let i = 0; i < nonNullishPairs.length; i++) { const pair = nonNullishPairs[i]; const value = JSON.stringify(pair[1], replacer); patchQueryArguments.push([pair[0], value]); - const patches = getReplacePatches(pair[0], pair[1]); - const [sql, args] = generateJSONReplaceSQLBatch(patches); - if (args.length > 0) { - replaceQueryArguments.push(...args); + const patches = mergeReplaceNullPatches?.[pair[0]] ?? []; + if (patches.length > 0) { + const queries = generateJSONReplaceSQLQueries(pair[0], patches); + + if (queries.length > 0) { + replaceQueryArguments.push(...queries); + } } } diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index c189cf377..6304d0866 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -1,5 +1,5 @@ import type {BatchQueryResult, QueryResult} from 'react-native-quick-sqlite'; -import type {OnyxKey, OnyxValue} from '../../types'; +import type {MixedOperationsQueue, OnyxKey, OnyxValue} from '../../types'; type KeyValuePair = [OnyxKey, OnyxValue]; type KeyList = OnyxKey[]; @@ -39,7 +39,7 @@ type StorageProvider = { /** * Multiple merging of existing and new values in a batch */ - multiMerge: (pairs: KeyValuePairList) => Promise; + multiMerge: (pairs: KeyValuePairList, mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches']) => Promise; /** * Merges an existing value with a new one diff --git a/lib/types.ts b/lib/types.ts index c492d869a..cd5556b89 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -3,6 +3,7 @@ import type {BuiltIns} from 'type-fest/source/internal'; import type OnyxUtils from './OnyxUtils'; import type {WithOnyxInstance, WithOnyxState} from './withOnyx/types'; import type {OnyxMethod} from './OnyxUtils'; +import type {FastMergeReplaceNullPatch} from './utils'; /** * Utility type that excludes `null` from the type `TValue`. @@ -490,6 +491,7 @@ type GenericFunction = (...args: any[]) => any; */ type MixedOperationsQueue = { merge: OnyxInputKeyValueMapping; + mergeReplaceNullPatches: {[TKey in OnyxKey]: FastMergeReplaceNullPatch[]}; set: OnyxInputKeyValueMapping; }; diff --git a/lib/utils.ts b/lib/utils.ts index 566c5797c..88091f6cd 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -5,6 +5,17 @@ import type {ConnectOptions, OnyxInput, OnyxKey} from './types'; type EmptyObject = Record; type EmptyValue = EmptyObject | null | undefined; +type FastMergeReplaceNullPatch = [string[], unknown]; + +type FastMergeMetadata = { + replaceNullPatches: FastMergeReplaceNullPatch[]; +}; + +type FastMergeResult = { + result: TValue; + replaceNullPatches: FastMergeReplaceNullPatch[]; +}; + const ONYX_INTERNALS__REPLACE_OBJECT_MARK = 'ONYX_INTERNALS__REPLACE_OBJECT_MARK'; /** Checks whether the given object is an object and not null/undefined. */ @@ -39,6 +50,8 @@ function mergeObject>( shouldRemoveNestedNulls: boolean, isBatchingMergeChanges: boolean, shouldReplaceMarkedObjects: boolean, + metadata: FastMergeMetadata, + basePath: string[] = [], ): TObject { const destination: Record = {}; @@ -98,6 +111,7 @@ function mergeObject>( // effectively replace them in the next condition. if (isBatchingMergeChanges && targetValue === null) { (targetValueWithFallback as Record)[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; + metadata.replaceNullPatches.push([[...basePath, key], {...sourceValue}]); } // Then, when merging the batched changes with the Onyx value, if a nested object of the batched changes @@ -110,7 +124,10 @@ function mergeObject>( delete (destination[key] as Record).ONYX_INTERNALS__REPLACE_OBJECT_MARK; } else { // For the normal situations we'll just call `fastMerge()` again to merge the nested object. - destination[key] = fastMerge(targetValueWithFallback, sourceValue, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects); + destination[key] = fastMerge(targetValueWithFallback, sourceValue, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects, metadata, [ + ...basePath, + key, + ]).result; } } else { destination[key] = sourceValue; @@ -126,22 +143,38 @@ function mergeObject>( * * We generally want to remove null values from objects written to disk and cache, because it decreases the amount of data stored in memory and on disk. */ -function fastMerge(target: TValue, source: TValue, shouldRemoveNestedNulls: boolean, isBatchingMergeChanges: boolean, shouldReplaceMarkedObjects: boolean): TValue { +function fastMerge( + target: TValue, + source: TValue, + shouldRemoveNestedNulls: boolean, + isBatchingMergeChanges: boolean, + shouldReplaceMarkedObjects: boolean, + metadata?: FastMergeMetadata, + basePath: string[] = [], +): FastMergeResult { + if (!metadata) { + // eslint-disable-next-line no-param-reassign + metadata = { + replaceNullPatches: [], + }; + } + // We have to ignore arrays and nullish values here, // otherwise "mergeObject" will throw an error, // because it expects an object as "source" if (Array.isArray(source) || source === null || source === undefined) { - return source; + return {result: source, replaceNullPatches: metadata.replaceNullPatches}; } - return mergeObject(target, source as Record, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects) as TValue; + const mergedValue = mergeObject(target, source as Record, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects, metadata, basePath) as TValue; + + return {result: mergedValue, replaceNullPatches: metadata.replaceNullPatches}; } /** Deep removes the nested null values from the given value. */ function removeNestedNullValues | null>(value: TValue): TValue { if (typeof value === 'object' && !Array.isArray(value)) { - const objectValue = value as Record; - return fastMerge(objectValue, objectValue, true, false, true) as TValue; + return fastMerge(value, value, true, false, true).result; } return value; @@ -245,3 +278,4 @@ export default { hasWithOnyxInstance, ONYX_INTERNALS__REPLACE_OBJECT_MARK, }; +export type {FastMergeResult, FastMergeReplaceNullPatch}; diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index 01614f076..b70a2d56f 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -56,7 +56,7 @@ describe('fastMerge', () => { it('should merge an object with another object and remove nested null values', () => { const result = utils.fastMerge(testObject, testObjectWithNullishValues, true, false, false); - expect(result).toEqual({ + expect(result.result).toEqual({ a: 'a', b: { c: { @@ -73,7 +73,7 @@ describe('fastMerge', () => { it('should merge an object with another object and not remove nested null values', () => { const result = utils.fastMerge(testObject, testObjectWithNullishValues, false, false, false); - expect(result).toEqual({ + expect(result.result).toEqual({ a: 'a', b: { c: { @@ -91,7 +91,7 @@ describe('fastMerge', () => { it('should merge an object with an empty object and remove deeply nested null values', () => { const result = utils.fastMerge({}, testObjectWithNullishValues, true, false, false); - expect(result).toEqual(testObjectWithNullValuesRemoved); + expect(result.result).toEqual(testObjectWithNullValuesRemoved); }); it('should remove null values by merging two identical objects with fastMerge', () => { @@ -104,20 +104,20 @@ describe('fastMerge', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const result = utils.fastMerge(testObject, [1, 2, 3] as any, true, false, false); - expect(result).toEqual([1, 2, 3]); + expect(result.result).toEqual([1, 2, 3]); }); it('should replace an array with an object', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const result = utils.fastMerge([1, 2, 3] as any, testObject, true, false, false); - expect(result).toEqual(testObject); + expect(result.result).toEqual(testObject); }); it('should add the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag to the target object when its source is set to null and "isBatchingMergeChanges" is true', () => { const result = utils.fastMerge(testMergeChanges[1], testMergeChanges[0], true, true, false); - expect(result).toEqual({ + expect(result.result).toEqual({ b: { d: { h: 'h', @@ -126,6 +126,7 @@ describe('fastMerge', () => { h: 'h', }, }); + expect(result.replaceNullPatches).toEqual([[['b', 'd'], {h: 'h'}]]); }); it('should completely replace the target object with its source when the source has the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag and "shouldReplaceMarkedObjects" is true', () => { @@ -145,7 +146,7 @@ describe('fastMerge', () => { true, ); - expect(result).toEqual({ + expect(result.result).toEqual({ a: 'a', b: { c: 'c', diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index 7479ced74..d16c0f169 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -203,7 +203,7 @@ describe('OnyxUtils', () => { it('should apply the replacement markers if the we have properties with objects being removed and added back during the changes', () => { const result = OnyxUtils.batchMergeChanges(testMergeChanges); - expect(result).toEqual({ + expect(result.result).toEqual({ b: { d: { i: 'i', @@ -217,6 +217,61 @@ describe('OnyxUtils', () => { }, }, }); + expect(result.replaceNullPatches).toEqual([ + [['b', 'd'], {i: 'i'}], + [['b', 'd'], {i: 'i', j: 'j'}], + [['b', 'g'], {k: 'k'}], + ]); + }); + + it('should 2', () => { + const result = OnyxUtils.batchMergeChanges([ + { + // Removing the "originalMessage" object in this update. + // Any subsequent changes to this object should completely replace the existing object in store. + originalMessage: null, + }, + { + // This change should completely replace "originalMessage" existing object in store. + originalMessage: { + errorMessage: 'newErrorMessage', + }, + receipt: { + // Removing the "nestedObject" object in this update. + // Any subsequent changes to this object should completely replace the existing object in store. + nestedObject: null, + }, + }, + { + receipt: { + receiptID: null, + filename: 'newFilename', + // This change should completely replace "receipt" existing object in store. + nestedObject: { + nestedKey2: 'newNestedKey2', + }, + }, + }, + ]); + + expect(result.result).toEqual({ + originalMessage: { + errorMessage: 'newErrorMessage', + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + }, + receipt: { + receiptID: null, + filename: 'newFilename', + nestedObject: { + nestedKey2: 'newNestedKey2', + [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, + }, + }, + }); + expect(result.replaceNullPatches).toEqual([ + [['originalMessage'], {errorMessage: 'newErrorMessage'}], + [['receipt', 'nestedObject'], {nestedKey2: 'newNestedKey2'}], + ]); }); }); }); From 01e9ff4236c3c25ab533823ea3d7b21241329ca9 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 15 May 2025 16:22:50 +0200 Subject: [PATCH 17/58] fix: improve fastMerge code --- lib/utils.ts | 231 +++++++++++++++++++++++++++++---------------------- 1 file changed, 132 insertions(+), 99 deletions(-) diff --git a/lib/utils.ts b/lib/utils.ts index 88091f6cd..fbb6c3ab2 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -7,51 +7,76 @@ type EmptyValue = EmptyObject | null | undefined; type FastMergeReplaceNullPatch = [string[], unknown]; +type FastMergeOptions = { + /** If true, null object values will be removed. */ + shouldRemoveNestedNulls?: boolean; + /** If true, it means that we are batching merge changes before applying them to the Onyx value, so we must use a special logic to handle these changes. */ + isBatchingMergeChanges?: boolean; + /** If true, any nested objects that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag will be completely replaced instead of merged. */ + shouldReplaceMarkedObjects?: boolean; +}; + type FastMergeMetadata = { + /** The path to the object that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag. */ replaceNullPatches: FastMergeReplaceNullPatch[]; }; type FastMergeResult = { + /** The result of the merge. */ result: TValue; + /** The path to the object that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag. */ replaceNullPatches: FastMergeReplaceNullPatch[]; }; const ONYX_INTERNALS__REPLACE_OBJECT_MARK = 'ONYX_INTERNALS__REPLACE_OBJECT_MARK'; -/** Checks whether the given object is an object and not null/undefined. */ -function isEmptyObject(obj: T | EmptyValue): obj is EmptyValue { - return typeof obj === 'object' && Object.keys(obj || {}).length === 0; -} - -// Mostly copied from https://medium.com/@lubaka.a/how-to-remove-lodash-performance-improvement-b306669ad0e1 - /** - * Checks whether the given value can be merged. It has to be an object, but not an array, RegExp or Date. + * Merges two objects and removes null values if "shouldRemoveNestedNulls" is set to true + * + * We generally want to remove null values from objects written to disk and cache, because it decreases the amount of data stored in memory and on disk. */ -function isMergeableObject(value: unknown): value is Record { - const isNonNullObject = value != null ? typeof value === 'object' : false; - return isNonNullObject && !(value instanceof RegExp) && !(value instanceof Date) && !Array.isArray(value); +function fastMerge(target: TValue, source: TValue, options?: FastMergeOptions, metadata?: FastMergeMetadata, basePath: string[] = []): FastMergeResult { + if (!metadata) { + // eslint-disable-next-line no-param-reassign + metadata = { + replaceNullPatches: [], + }; + } + + // We have to ignore arrays and nullish values here, + // otherwise "mergeObject" will throw an error, + // because it expects an object as "source" + if (Array.isArray(source) || source === null || source === undefined) { + return {result: source, replaceNullPatches: metadata.replaceNullPatches}; + } + + const optionsWithDefaults = { + shouldRemoveNestedNulls: false, + isBatchingMergeChanges: false, + shouldReplaceMarkedObjects: false, + ...options, + }; + + const mergedValue = mergeObject(target, source as Record, optionsWithDefaults, metadata, basePath) as TValue; + + return {result: mergedValue, replaceNullPatches: metadata.replaceNullPatches}; } /** * Merges the source object into the target object. * @param target - The target object. * @param source - The source object. - * @param shouldRemoveNestedNulls - If true, null object values will be removed. - * @param isBatchingMergeChanges - If true, it means that we are batching merge changes before applying - * them to the Onyx value, so we must use a special logic to handle these changes. - * @param shouldReplaceMarkedObjects - If true, any nested objects that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" - * flag will be completely replaced instead of merged. + * @param options - The options for the merge. + * @param metadata - The metadata for the merge. + * @param basePath - The base path for the merge. * @returns - The merged object. */ function mergeObject>( target: TObject | unknown | null | undefined, source: TObject, - shouldRemoveNestedNulls: boolean, - isBatchingMergeChanges: boolean, - shouldReplaceMarkedObjects: boolean, + options: FastMergeOptions, metadata: FastMergeMetadata, - basePath: string[] = [], + basePath: string[], ): TObject { const destination: Record = {}; @@ -64,19 +89,20 @@ function mergeObject>( if (targetObject) { // eslint-disable-next-line no-restricted-syntax, guard-for-in for (const key in targetObject) { - const sourceValue = source?.[key]; - const targetValue = targetObject?.[key]; + const targetProperty = targetObject?.[key]; + if (targetProperty === undefined) { + // eslint-disable-next-line no-continue + continue; + } - // If "shouldRemoveNestedNulls" is true, we want to remove null values from the merged object. - // Therefore, if either target or source value is null, we want to prevent the key from being set. - // targetValue should techincally never be "undefined", because it will always be a value from cache or storage - // and we never set "undefined" there. Still, if there targetValue is undefined we don't want to set - // the key explicitly to prevent loose undefined values in objects in cache and storage. - const isSourceOrTargetNull = targetValue === undefined || targetValue === null || sourceValue === null; - const shouldOmitTargetKey = shouldRemoveNestedNulls && isSourceOrTargetNull; + // If "shouldRemoveNestedNulls" is true, we want to remove (nested) null values from the merged object. + // If either the source or target value is null, we want to omit the key from the merged object. + const sourceProperty = source?.[key]; + const isSourceOrTargetNull = targetProperty === null || sourceProperty === null; + const shouldOmitTargetKey = options.shouldRemoveNestedNulls && isSourceOrTargetNull; if (!shouldOmitTargetKey) { - destination[key] = targetValue; + destination[key] = targetProperty; } } } @@ -84,97 +110,104 @@ function mergeObject>( // After copying over all keys from the target object, we want to merge the source object into the destination object. // eslint-disable-next-line no-restricted-syntax, guard-for-in for (const key in source) { - const sourceValue = source?.[key] as Record; - const targetValue = targetObject?.[key]; + const sourceProperty = source?.[key] as Record; + if (sourceProperty === undefined) { + // eslint-disable-next-line no-continue + continue; + } - // If undefined is passed as the source value for a key, we want to generally ignore it. // If "shouldRemoveNestedNulls" is set to true and the source value is null, // we don't want to set/merge the source value into the merged object. - const shouldIgnoreNullSourceValue = shouldRemoveNestedNulls && sourceValue === null; - const shouldOmitSourceKey = sourceValue === undefined || shouldIgnoreNullSourceValue; + const shouldOmitSourceKey = options.shouldRemoveNestedNulls && sourceProperty === null; + if (shouldOmitSourceKey) { + // eslint-disable-next-line no-continue + continue; + } - if (!shouldOmitSourceKey) { - // If the source value is a mergable object, we want to merge it into the target value. - // If "shouldRemoveNestedNulls" is true, "fastMerge" will recursively - // remove nested null values from the merged object. + // If the source value is a mergable object, we want to merge it into the target value. + if (!isMergeableObject(sourceProperty)) { // If source value is any other value we need to set the source value it directly. - if (isMergeableObject(sourceValue)) { - // If the target value is null or undefined, we need to fallback to an empty object, - // so that we can still use "fastMerge" to merge the source value, - // to ensure that nested null values are removed from the merged object. - const targetValueWithFallback = (targetValue ?? {}) as TObject; - - // If we are batching merge changes and the previous merge change (targetValue) is null, - // it means we want to fully replace this object when merging the batched changes with the Onyx value. - // To achieve this, we first mark these nested objects with an internal flag. With the desired objects - // marked, when calling this method again with "shouldReplaceMarkedObjects" set to true we can proceed - // effectively replace them in the next condition. - if (isBatchingMergeChanges && targetValue === null) { - (targetValueWithFallback as Record)[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; - metadata.replaceNullPatches.push([[...basePath, key], {...sourceValue}]); - } - - // Then, when merging the batched changes with the Onyx value, if a nested object of the batched changes - // has the internal flag set, we replace the entire destination object with the source one and remove - // the flag. - if (shouldReplaceMarkedObjects && sourceValue[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { - // We do a spread here in order to have a new object reference and allow us to delete the internal flag - // of the merged object only. - destination[key] = {...sourceValue}; - delete (destination[key] as Record).ONYX_INTERNALS__REPLACE_OBJECT_MARK; - } else { - // For the normal situations we'll just call `fastMerge()` again to merge the nested object. - destination[key] = fastMerge(targetValueWithFallback, sourceValue, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects, metadata, [ - ...basePath, - key, - ]).result; - } - } else { - destination[key] = sourceValue; - } + destination[key] = sourceProperty; } + + const targetProperty = targetObject?.[key]; + const targetWithMarks = getTargetPropertyWithRemovalMark(targetProperty, sourceProperty, options, metadata, basePath); + const {finalDestinationProperty, stopTraversing} = replaceMarkedObjects(sourceProperty, options); + + if (stopTraversing) { + destination[key] = finalDestinationProperty; + // eslint-disable-next-line no-continue + continue; + } + + destination[key] = fastMerge(targetWithMarks, sourceProperty, options, metadata, [...basePath, key]).result; } return destination as TObject; } +/** Checks whether the given object is an object and not null/undefined. */ +function isEmptyObject(obj: T | EmptyValue): obj is EmptyValue { + return typeof obj === 'object' && Object.keys(obj || {}).length === 0; +} + +// Mostly copied from https://medium.com/@lubaka.a/how-to-remove-lodash-performance-improvement-b306669ad0e1 + /** - * Merges two objects and removes null values if "shouldRemoveNestedNulls" is set to true - * - * We generally want to remove null values from objects written to disk and cache, because it decreases the amount of data stored in memory and on disk. + * Checks whether the given value can be merged. It has to be an object, but not an array, RegExp or Date. */ -function fastMerge( - target: TValue, - source: TValue, - shouldRemoveNestedNulls: boolean, - isBatchingMergeChanges: boolean, - shouldReplaceMarkedObjects: boolean, - metadata?: FastMergeMetadata, +function isMergeableObject>(value: unknown): value is TObject { + const isNonNullObject = value != null ? typeof value === 'object' : false; + return isNonNullObject && !(value instanceof RegExp) && !(value instanceof Date) && !Array.isArray(value); +} + +function getTargetPropertyWithRemovalMark>( + targetProperty: unknown, + sourceProperty: Record, + options: FastMergeOptions, + metadata: FastMergeMetadata, basePath: string[] = [], -): FastMergeResult { - if (!metadata) { - // eslint-disable-next-line no-param-reassign - metadata = { - replaceNullPatches: [], - }; +): TObject { + const targetPropertyWithMarks = (targetProperty ?? {}) as Record; + + // If we are batching merge changes and the previous merge change (targetValue) is null, + // it means we want to fully replace this object when merging the batched changes with the Onyx value. + // To achieve this, we first mark these nested objects with an internal flag. With the desired objects + // marked, when calling this method again with "shouldReplaceMarkedObjects" set to true we can proceed + // effectively replace them in the next condition. + if (options?.isBatchingMergeChanges && targetProperty === null) { + targetPropertyWithMarks[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; + metadata.replaceNullPatches.push([[...basePath], {...sourceProperty}]); } - // We have to ignore arrays and nullish values here, - // otherwise "mergeObject" will throw an error, - // because it expects an object as "source" - if (Array.isArray(source) || source === null || source === undefined) { - return {result: source, replaceNullPatches: metadata.replaceNullPatches}; - } + return targetPropertyWithMarks as TObject; +} - const mergedValue = mergeObject(target, source as Record, shouldRemoveNestedNulls, isBatchingMergeChanges, shouldReplaceMarkedObjects, metadata, basePath) as TValue; +function replaceMarkedObjects>(sourceProperty: TObject, options: FastMergeOptions): {finalDestinationProperty?: TObject; stopTraversing: boolean} { + // Then, when merging the batched changes with the Onyx value, if a nested object of the batched changes + // has the internal flag set, we replace the entire destination object with the source one and remove + // the flag. + if (options.shouldReplaceMarkedObjects && sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { + // We do a spread here in order to have a new object reference and allow us to delete the internal flag + // of the merged object only. + + const destinationProperty = {...sourceProperty}; + delete destinationProperty.ONYX_INTERNALS__REPLACE_OBJECT_MARK; + return {finalDestinationProperty: destinationProperty, stopTraversing: true}; + } - return {result: mergedValue, replaceNullPatches: metadata.replaceNullPatches}; + // For the normal situations we'll just call `fastMerge()` again to merge the nested object. + return {stopTraversing: false}; } /** Deep removes the nested null values from the given value. */ function removeNestedNullValues | null>(value: TValue): TValue { if (typeof value === 'object' && !Array.isArray(value)) { - return fastMerge(value, value, true, false, true).result; + return fastMerge(value, value, { + shouldRemoveNestedNulls: true, + isBatchingMergeChanges: false, + shouldReplaceMarkedObjects: false, + }).result; } return value; @@ -268,8 +301,8 @@ function hasWithOnyxInstance(mapping: ConnectOptions } export default { - isEmptyObject, fastMerge, + isEmptyObject, formatActionName, removeNestedNullValues, checkCompatibilityWithExistingValue, From a08b5e4f81d881d6f74f9491ba7ef7a2377c1f1d Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 15 May 2025 16:24:41 +0200 Subject: [PATCH 18/58] adapt fastMerge usages --- lib/OnyxCache.ts | 7 +++++- lib/OnyxUtils.ts | 17 +++++++++++---- lib/storage/providers/IDBKeyValProvider.ts | 5 ++++- lib/storage/providers/MemoryOnlyProvider.ts | 5 ++++- tests/perf-test/utils.perf-test.ts | 7 +++++- tests/unit/fastMergeTest.ts | 24 +++++++++++++++------ 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index 691cedfee..d0d4585d4 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -164,7 +164,12 @@ class OnyxCache { throw new Error('data passed to cache.merge() must be an Object of onyx key/value pairs'); } - this.storageMap = {...utils.fastMerge(this.storageMap, data, true, false, true).result}; + this.storageMap = { + ...utils.fastMerge(this.storageMap, data, { + shouldRemoveNestedNulls: true, + shouldReplaceMarkedObjects: true, + }).result, + }; Object.entries(data).forEach(([key, value]) => { this.addKey(key); diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 93b6bf651..52452f931 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1265,7 +1265,14 @@ function applyMerge | undefined, TChange exten if (changes.some((change) => change && typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, true, false, true).result, (existingValue || {}) as TChange); + return changes.reduce( + (modifiedData, change) => + utils.fastMerge(modifiedData, change, { + shouldRemoveNestedNulls: true, + shouldReplaceMarkedObjects: true, + }).result, + (existingValue || {}) as TChange, + ); } // If we have anything else we can't merge it so we'll @@ -1284,7 +1291,7 @@ function batchMergeChanges | undefined>(chang // Object values are then merged one after the other return changes.reduce>( (modifiedData, change) => { - const fastMergeResult = utils.fastMerge(modifiedData.result, change, false, true, false); + const fastMergeResult = utils.fastMerge(modifiedData.result, change, {isBatchingMergeChanges: true}); // eslint-disable-next-line no-param-reassign modifiedData.result = fastMergeResult.result; // eslint-disable-next-line no-param-reassign @@ -1310,7 +1317,9 @@ function initializeWithDefaultKeyStates(): Promise { return Storage.multiGet(Object.keys(defaultKeyStates)).then((pairs) => { const existingDataAsObject = Object.fromEntries(pairs); - const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, true, false, false).result; + const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, { + shouldRemoveNestedNulls: true, + }).result; cache.merge(merged ?? {}); Object.entries(merged ?? {}).forEach(([key, value]) => keyChanged(key, value, existingDataAsObject)); @@ -1373,7 +1382,7 @@ function subscribeToKey(connectOptions: ConnectOptions { const prev = values[index]; - const newValue = utils.fastMerge(prev as Record, value as Record, true, false, true).result; + const newValue = utils.fastMerge(prev as Record, value as Record, { + shouldRemoveNestedNulls: true, + shouldReplaceMarkedObjects: true, + }).result; return promisifyRequest(store.put(newValue, key)); }); return Promise.all(upsertMany); diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index 7f1b1c2fb..3ec4d6b56 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -86,7 +86,10 @@ const provider: StorageProvider = { multiMerge(pairs) { _.forEach(pairs, ([key, value]) => { const existingValue = store[key] as Record; - const newValue = utils.fastMerge(existingValue, value as Record, true, false, true).result as OnyxValue; + const newValue = utils.fastMerge(existingValue, value as Record, { + shouldRemoveNestedNulls: true, + shouldReplaceMarkedObjects: true, + }).result as OnyxValue; set(key, newValue); }); diff --git a/tests/perf-test/utils.perf-test.ts b/tests/perf-test/utils.perf-test.ts index b3ab893c7..e0ada9009 100644 --- a/tests/perf-test/utils.perf-test.ts +++ b/tests/perf-test/utils.perf-test.ts @@ -15,6 +15,11 @@ describe('[Utils.js]', () => { const target = getMockedPersonalDetails(1000); const source = getMockedPersonalDetails(500); - await measureFunction(() => utils.fastMerge(target, source, true, false, false)); + await measureFunction(() => + utils.fastMerge(target, source, { + shouldRemoveNestedNulls: true, + shouldReplaceMarkedObjects: true, + }), + ); }); }); diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index b70a2d56f..f824d10d7 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -89,7 +89,9 @@ describe('fastMerge', () => { }); it('should merge an object with an empty object and remove deeply nested null values', () => { - const result = utils.fastMerge({}, testObjectWithNullishValues, true, false, false); + const result = utils.fastMerge({}, testObjectWithNullishValues, { + shouldRemoveNestedNulls: true, + }); expect(result.result).toEqual(testObjectWithNullValuesRemoved); }); @@ -102,20 +104,27 @@ describe('fastMerge', () => { it('should replace an object with an array', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = utils.fastMerge(testObject, [1, 2, 3] as any, true, false, false); + const result = utils.fastMerge(testObject, [1, 2, 3] as any, { + shouldRemoveNestedNulls: true, + }); expect(result.result).toEqual([1, 2, 3]); }); it('should replace an array with an object', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = utils.fastMerge([1, 2, 3] as any, testObject, true, false, false); + const result = utils.fastMerge([1, 2, 3] as any, testObject, { + shouldRemoveNestedNulls: true, + }); expect(result.result).toEqual(testObject); }); it('should add the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag to the target object when its source is set to null and "isBatchingMergeChanges" is true', () => { - const result = utils.fastMerge(testMergeChanges[1], testMergeChanges[0], true, true, false); + const result = utils.fastMerge(testMergeChanges[1], testMergeChanges[0], { + shouldRemoveNestedNulls: true, + isBatchingMergeChanges: true, + }); expect(result.result).toEqual({ b: { @@ -141,9 +150,10 @@ describe('fastMerge', () => { h: 'h', }, }, - true, - false, - true, + { + shouldRemoveNestedNulls: true, + shouldReplaceMarkedObjects: true, + }, ); expect(result.result).toEqual({ From d89bdd095abf842ae2eb0d8ab0eb6bf848359a4f Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 15 May 2025 17:06:28 +0200 Subject: [PATCH 19/58] combine back batchMergeChanges and applyMerge --- API-INTERNAL.md | 10 +++++----- lib/OnyxUtils.ts | 37 +++++++------------------------------ tests/unit/onyxUtilsTest.ts | 14 ++++++-------- 3 files changed, 18 insertions(+), 43 deletions(-) diff --git a/API-INTERNAL.md b/API-INTERNAL.md index d8665de66..a098586b3 100644 --- a/API-INTERNAL.md +++ b/API-INTERNAL.md @@ -149,8 +149,8 @@ if shouldRemoveNestedNulls is true and returns the object.

This method transforms an object like {'@MyApp_user': myUserValue, '@MyApp_key': myKeyValue} to an array of key-value pairs in the above format and removes key-value pairs that are being set to null

-
applyMerge(changes)
-

Merges an array of changes with an existing value

+
mergeChanges(changes)
+

Merges an array of changes with an existing value or creates a single change

initializeWithDefaultKeyStates()

Merge user provided default key value pairs.

@@ -483,10 +483,10 @@ to an array of key-value pairs in the above format and removes key-value pairs t **Kind**: global function **Returns**: an array of key - value pairs <[key, value]> - + -## applyMerge(changes) -Merges an array of changes with an existing value +## mergeChanges(changes) +Merges an array of changes with an existing value or creates a single change **Kind**: global function diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 52452f931..98017bd33 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1252,35 +1252,12 @@ function prepareKeyValuePairsForStorage(data: Record } /** - * Merges an array of changes with an existing value + * Merges an array of changes with an existing value or creates a single change * - * @param changes Array of changes that should be applied to the existing value + * @param changes Array of changes that should be merged + * @param existingValue The existing value that should be merged with the changes */ -function applyMerge | undefined, TChange extends OnyxInput | undefined>(existingValue: TValue, changes: TChange[]): TChange { - const lastChange = changes?.at(-1); - - if (Array.isArray(lastChange)) { - return lastChange; - } - - if (changes.some((change) => change && typeof change === 'object')) { - // Object values are then merged one after the other - return changes.reduce( - (modifiedData, change) => - utils.fastMerge(modifiedData, change, { - shouldRemoveNestedNulls: true, - shouldReplaceMarkedObjects: true, - }).result, - (existingValue || {}) as TChange, - ); - } - - // If we have anything else we can't merge it so we'll - // simply return the last value that was queued - return lastChange as TChange; -} - -function batchMergeChanges | undefined>(changes: TChange[]): FastMergeResult { +function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { @@ -1299,7 +1276,7 @@ function batchMergeChanges | undefined>(chang return modifiedData; }, { - result: {} as TChange, + result: (existingValue ?? {}) as TChange, replaceNullPatches: [], }, ); @@ -1319,6 +1296,7 @@ function initializeWithDefaultKeyStates(): Promise { const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, { shouldRemoveNestedNulls: true, + shouldReplaceMarkedObjects: true, }).result; cache.merge(merged ?? {}); @@ -1509,7 +1487,7 @@ const OnyxUtils = { hasPendingMergeForKey, removeNullValues, prepareKeyValuePairsForStorage, - applyMerge, + mergeChanges, initializeWithDefaultKeyStates, getSnapshotKey, multiGet, @@ -1521,7 +1499,6 @@ const OnyxUtils = { getEvictionBlocklist, getSkippableCollectionMemberIDs, setSkippableCollectionMemberIDs, - batchMergeChanges, }; GlobalSettings.addGlobalSettingsChangeListener(({enablePerformanceMetrics}) => { diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index d16c0f169..c017ef3d8 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -151,15 +151,15 @@ describe('OnyxUtils', () => { }); }); - describe('applyMerge', () => { + describe('mergeChanges', () => { it("should return the last change if it's an array", () => { - const result = OnyxUtils.applyMerge(testObject, [...testMergeChanges, [0, 1, 2]]); + const result = OnyxUtils.mergeChanges([...testMergeChanges, [0, 1, 2]], testObject); expect(result).toEqual([0, 1, 2]); }); it("should return the last change if the changes aren't objects", () => { - const result = OnyxUtils.applyMerge(testObject, ['a', 0, 'b', 1]); + const result = OnyxUtils.mergeChanges(['a', 0, 'b', 1], testObject); expect(result).toEqual(1); }); @@ -180,7 +180,7 @@ describe('OnyxUtils', () => { }, }; - const result = OnyxUtils.applyMerge(testObject, [batchedChanges]); + const result = OnyxUtils.mergeChanges([batchedChanges], testObject); expect(result).toEqual({ a: 'a', @@ -197,11 +197,9 @@ describe('OnyxUtils', () => { }, }); }); - }); - describe('batchMergeChanges', () => { it('should apply the replacement markers if the we have properties with objects being removed and added back during the changes', () => { - const result = OnyxUtils.batchMergeChanges(testMergeChanges); + const result = OnyxUtils.mergeChanges(testMergeChanges); expect(result.result).toEqual({ b: { @@ -225,7 +223,7 @@ describe('OnyxUtils', () => { }); it('should 2', () => { - const result = OnyxUtils.batchMergeChanges([ + const result = OnyxUtils.mergeChanges([ { // Removing the "originalMessage" object in this update. // Any subsequent changes to this object should completely replace the existing object in store. From 43e6665daccfa97f9d3ae867733883d614a15d56 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 15 May 2025 17:06:55 +0200 Subject: [PATCH 20/58] fix: simplify Onyx.merge --- lib/Onyx.ts | 41 ++++++++++++----------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index c89d24f38..cdb04f59d 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -41,7 +41,7 @@ function init({ initialKeyStates = {}, safeEvictionKeys = [], maxCachedKeysCount = 1000, - shouldSyncMultipleInstances = Boolean(global.localStorage), + shouldSyncMultipleInstances = !!global.localStorage, debugSetState = false, enablePerformanceMetrics = false, skippableCollectionMemberIDs = [], @@ -319,54 +319,37 @@ function merge(key: TKey, changes: OnyxMergeInput): if (!validChanges.length) { return Promise.resolve(); } - const batchedDeltaChanges = OnyxUtils.batchMergeChanges(validChanges).result; - - // Case (1): When there is no existing value in storage, we want to set the value instead of merge it. - // Case (2): The presence of a top-level `null` in the merge queue instructs us to drop the whole existing value. - // In this case, we can't simply merge the batched changes with the existing value, because then the null in the merge queue would have no effect. - const shouldSetValue = !existingValue || mergeQueue[key].includes(null); // Clean up the write queue, so we don't apply these changes again. delete mergeQueue[key]; delete mergeQueuePromise[key]; - const logMergeCall = (hasChanged = true) => { - // Logging properties only since values could be sensitive things we don't want to log. - Logger.logInfo(`merge called for key: ${key}${_.isObject(batchedDeltaChanges) ? ` properties: ${_.keys(batchedDeltaChanges).join(',')}` : ''} hasChanged: ${hasChanged}`); - }; - - // If the batched changes equal null, we want to remove the key from storage, to reduce storage size. - const {wasRemoved} = OnyxUtils.removeNullValues(key, batchedDeltaChanges); - - // Calling "OnyxUtils.removeNullValues" removes the key from storage and cache and updates the subscriber. + // Calling "OnyxUtils.remove" removes the key from storage and cache and updates the subscriber. // Therefore, we don't need to further broadcast and update the value so we can return early. - if (wasRemoved) { - logMergeCall(); + if (validChanges.at(-1) === null) { + Logger.logInfo(`merge called for key: ${key} was removed`); + OnyxUtils.remove(key); return Promise.resolve(); } - // If "shouldSetValue" is true, it means that we want to completely replace the existing value with the batched changes, - // so we pass `undefined` to OnyxUtils.applyMerge() first parameter to make it use "batchedDeltaChanges" to - // create a new object for us. - // If "shouldSetValue" is false, it means that we want to merge the batched changes into the existing value, - // so we pass "existingValue" to the first parameter. - const resultValue = OnyxUtils.applyMerge(shouldSetValue ? undefined : existingValue, [batchedDeltaChanges]); + const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. - const hasChanged = cache.hasValueChanged(key, resultValue); + const hasChanged = cache.hasValueChanged(key, mergedValue); - logMergeCall(hasChanged); + // Logging properties only since values could be sensitive things we don't want to log. + Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`); // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. - const updatePromise = OnyxUtils.broadcastUpdate(key, resultValue as OnyxValue, hasChanged); + const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue, hasChanged); // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. if (!hasChanged) { return updatePromise; } - return Storage.mergeItem(key, resultValue as OnyxValue).then(() => { - OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, resultValue); + return Storage.setItem(key, mergedValue as OnyxValue).then(() => { + OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, mergedValue); return updatePromise; }); } catch (error) { From f190fbd46efb2c67faf0b338beeb6db4801d9cf0 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 15 May 2025 17:07:17 +0200 Subject: [PATCH 21/58] fix: improve Onyx.update --- lib/Onyx.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index cdb04f59d..81c1da503 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -757,7 +757,7 @@ function update(data: OnyxUpdate[]): Promise { // Remove the collection-related key from the updateQueue so that it won't be processed individually. delete updateQueue[key]; - const batchedChanges = OnyxUtils.batchMergeChanges(operations); + const batchedChanges = OnyxUtils.mergeChanges(operations); if (operations[0] === null) { // eslint-disable-next-line no-param-reassign queue.set[key] = batchedChanges.result; @@ -789,13 +789,16 @@ function update(data: OnyxUpdate[]): Promise { }); Object.entries(updateQueue).forEach(([key, operations]) => { - const batchedChanges = OnyxUtils.batchMergeChanges(operations).result; - if (operations[0] === null) { + const batchedChanges = OnyxUtils.mergeChanges(operations).result; promises.push(() => set(key, batchedChanges)); - } else { - promises.push(() => merge(key, batchedChanges)); + return; } + + const mergePromises = operations.map((operation) => { + return merge(key, operation); + }); + promises.push(() => mergePromises.at(0) ?? Promise.resolve()); }); const snapshotPromises = updateSnapshots(data); From 2701874d8ecf876cb8b608deda1219a0034166f1 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Thu, 15 May 2025 17:07:30 +0200 Subject: [PATCH 22/58] fix: change signature of storage.mergeItem --- lib/storage/index.ts | 4 ++-- lib/storage/providers/IDBKeyValProvider.ts | 4 ++-- lib/storage/providers/MemoryOnlyProvider.ts | 4 ++-- lib/storage/providers/SQLiteProvider.ts | 6 +++--- lib/storage/providers/types.ts | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/storage/index.ts b/lib/storage/index.ts index 33befed1d..f9b9e2d57 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -116,9 +116,9 @@ const storage: Storage = { /** * Merging an existing value with a new one */ - mergeItem: (key, preMergedValue) => + mergeItem: (key, change) => tryOrDegradePerformance(() => { - const promise = provider.mergeItem(key, preMergedValue); + const promise = provider.mergeItem(key, change); if (shouldKeepInstancesSync) { return promise.then(() => InstanceSync.mergeItem(key)); diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index ac40eb2f1..f4062304b 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -58,9 +58,9 @@ const provider: StorageProvider = { return Promise.all(upsertMany); }); }), - mergeItem(key, preMergedValue) { + mergeItem(key, change) { // Since Onyx already merged the existing value with the changes, we can just set the value directly. - return provider.setItem(key, preMergedValue); + return provider.multiMerge([[key, change]]); }, multiSet: (pairs) => { const pairsWithoutNull = pairs.filter(([key, value]) => { diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index 3ec4d6b56..de9a3acfd 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -74,9 +74,9 @@ const provider: StorageProvider = { /** * Merging an existing value with a new one */ - mergeItem(key, preMergedValue) { + mergeItem(key, change) { // Since Onyx already merged the existing value with the changes, we can just set the value directly. - return this.setItem(key, preMergedValue); + return this.multiMerge([[key, change]]); }, /** diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index d3dd7d7de..660a052de 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -3,7 +3,7 @@ * converting the value to a JSON string */ import {getFreeDiskStorage} from 'react-native-device-info'; -import type {BatchQueryResult, QuickSQLiteConnection, SQLBatchTuple} from 'react-native-quick-sqlite'; +import type {QuickSQLiteConnection, SQLBatchTuple} from 'react-native-quick-sqlite'; import {open} from 'react-native-quick-sqlite'; import type {FastMergeReplaceNullPatch} from '../../utils'; import utils from '../../utils'; @@ -114,9 +114,9 @@ const provider: StorageProvider = { return db.executeBatchAsync(commands); }, - mergeItem(key, preMergedValue) { + mergeItem(key, change) { // Since Onyx already merged the existing value with the changes, we can just set the value directly. - return this.setItem(key, preMergedValue) as Promise; + return this.multiMerge([[key, change]]); }, getAllKeys: () => db.executeAsync('SELECT record_key FROM keyvaluepairs;').then(({rows}) => { diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index 6304d0866..0a182a60e 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -43,9 +43,9 @@ type StorageProvider = { /** * Merges an existing value with a new one - * @param preMergedValue - the pre-merged data from `Onyx.applyMerge` + * @param change - the change to merge with the existing value */ - mergeItem: (key: TKey, preMergedValue: OnyxValue) => Promise; + mergeItem: (key: TKey, change: OnyxValue) => Promise; /** * Returns all keys available in storage From defed16fb9b7cc21bc07ff42c3b5a9e2a171c776 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 20 May 2025 13:08:20 +0200 Subject: [PATCH 23/58] refactor: remove return values from storage providers --- lib/storage/providers/IDBKeyValProvider.ts | 2 +- lib/storage/providers/MemoryOnlyProvider.ts | 2 +- lib/storage/providers/NoopProvider.ts | 2 +- lib/storage/providers/SQLiteProvider.ts | 12 +++++------ lib/storage/providers/types.ts | 22 ++++++++++++--------- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index f4062304b..036592d7b 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -55,7 +55,7 @@ const provider: StorageProvider = { }).result; return promisifyRequest(store.put(newValue, key)); }); - return Promise.all(upsertMany); + return Promise.all(upsertMany).then(() => undefined); }); }), mergeItem(key, change) { diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index de9a3acfd..a6016f99c 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -94,7 +94,7 @@ const provider: StorageProvider = { set(key, newValue); }); - return Promise.resolve([]); + return Promise.resolve(); }, /** diff --git a/lib/storage/providers/NoopProvider.ts b/lib/storage/providers/NoopProvider.ts index f99af069c..ccbee65a6 100644 --- a/lib/storage/providers/NoopProvider.ts +++ b/lib/storage/providers/NoopProvider.ts @@ -54,7 +54,7 @@ const provider: StorageProvider = { * This function also removes all nested null values from an object. */ multiMerge() { - return Promise.resolve([]); + return Promise.resolve(); }, /** diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 660a052de..fd3363f79 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -65,14 +65,14 @@ const provider: StorageProvider = { }); }, setItem(key, value) { - return db.executeAsync('REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, ?);', [key, JSON.stringify(value)]); + return db.executeAsync('REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, ?);', [key, JSON.stringify(value)]).then(() => undefined); }, multiSet(pairs) { const stringifiedPairs = pairs.map((pair) => [pair[0], JSON.stringify(pair[1] === undefined ? null : pair[1])]); if (utils.isEmptyObject(stringifiedPairs)) { return Promise.resolve(); } - return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]); + return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]).then(() => undefined); }, multiMerge(pairs, mergeReplaceNullPatches) { const commands: SQLBatchTuple[] = []; @@ -112,7 +112,7 @@ const provider: StorageProvider = { commands.push([replaceQuery, replaceQueryArguments]); } - return db.executeBatchAsync(commands); + return db.executeBatchAsync(commands).then(() => undefined); }, mergeItem(key, change) { // Since Onyx already merged the existing value with the changes, we can just set the value directly. @@ -124,13 +124,13 @@ const provider: StorageProvider = { const result = rows?._array.map((row) => row.record_key); return (result ?? []) as KeyList; }), - removeItem: (key) => db.executeAsync('DELETE FROM keyvaluepairs WHERE record_key = ?;', [key]), + removeItem: (key) => db.executeAsync('DELETE FROM keyvaluepairs WHERE record_key = ?;', [key]).then(() => undefined), removeItems: (keys) => { const placeholders = keys.map(() => '?').join(','); const query = `DELETE FROM keyvaluepairs WHERE record_key IN (${placeholders});`; - return db.executeAsync(query, keys); + return db.executeAsync(query, keys).then(() => undefined); }, - clear: () => db.executeAsync('DELETE FROM keyvaluepairs;', []), + clear: () => db.executeAsync('DELETE FROM keyvaluepairs;', []).then(() => undefined), getDatabaseSize() { return Promise.all([db.executeAsync('PRAGMA page_size;'), db.executeAsync('PRAGMA page_count;'), getFreeDiskStorage()]).then(([pageSizeResult, pageCountResult, bytesRemaining]) => { const pageSize: number = pageSizeResult.rows?.item(0).page_size; diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index 0a182a60e..fc3b3cade 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -1,10 +1,14 @@ -import type {BatchQueryResult, QueryResult} from 'react-native-quick-sqlite'; import type {MixedOperationsQueue, OnyxKey, OnyxValue} from '../../types'; type KeyValuePair = [OnyxKey, OnyxValue]; type KeyList = OnyxKey[]; type KeyValuePairList = KeyValuePair[]; +type DatabaseSize = { + bytesUsed: number; + bytesRemaining: number; +}; + type OnStorageKeyChanged = (key: TKey, value: OnyxValue) => void; type StorageProvider = { @@ -29,23 +33,23 @@ type StorageProvider = { /** * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string */ - setItem: (key: TKey, value: OnyxValue) => Promise; + setItem: (key: TKey, value: OnyxValue) => Promise; /** * Stores multiple key-value pairs in a batch */ - multiSet: (pairs: KeyValuePairList) => Promise; + multiSet: (pairs: KeyValuePairList) => Promise; /** * Multiple merging of existing and new values in a batch */ - multiMerge: (pairs: KeyValuePairList, mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches']) => Promise; + multiMerge: (pairs: KeyValuePairList, mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches']) => Promise; /** * Merges an existing value with a new one * @param change - the change to merge with the existing value */ - mergeItem: (key: TKey, change: OnyxValue) => Promise; + mergeItem: (key: TKey, change: OnyxValue) => Promise; /** * Returns all keys available in storage @@ -55,22 +59,22 @@ type StorageProvider = { /** * Removes given key and its value from storage */ - removeItem: (key: OnyxKey) => Promise; + removeItem: (key: OnyxKey) => Promise; /** * Removes given keys and their values from storage */ - removeItems: (keys: KeyList) => Promise; + removeItems: (keys: KeyList) => Promise; /** * Clears absolutely everything from storage */ - clear: () => Promise; + clear: () => Promise; /** * Gets the total bytes of the database file */ - getDatabaseSize: () => Promise<{bytesUsed: number; bytesRemaining: number}>; + getDatabaseSize: () => Promise; /** * @param onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync From a6312b16fc3fb362206600cbfce9748dc48a5a51 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 20 May 2025 13:11:31 +0200 Subject: [PATCH 24/58] fix: invalid fastMerge option --- tests/perf-test/utils.perf-test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/perf-test/utils.perf-test.ts b/tests/perf-test/utils.perf-test.ts index e0ada9009..6df36df08 100644 --- a/tests/perf-test/utils.perf-test.ts +++ b/tests/perf-test/utils.perf-test.ts @@ -18,7 +18,6 @@ describe('[Utils.js]', () => { await measureFunction(() => utils.fastMerge(target, source, { shouldRemoveNestedNulls: true, - shouldReplaceMarkedObjects: true, }), ); }); From 5f69d7a8bdac79c4ca175b19641b075b4960f3b4 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 20 May 2025 13:18:23 +0200 Subject: [PATCH 25/58] fix: TS error --- tests/unit/fastMergeTest.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index f824d10d7..4ed2cef6a 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -54,7 +54,7 @@ const testMergeChanges: DeepObject[] = [ describe('fastMerge', () => { it('should merge an object with another object and remove nested null values', () => { - const result = utils.fastMerge(testObject, testObjectWithNullishValues, true, false, false); + const result = utils.fastMerge(testObject, testObjectWithNullishValues, {shouldRemoveNestedNulls: true}); expect(result.result).toEqual({ a: 'a', @@ -71,7 +71,7 @@ describe('fastMerge', () => { }); it('should merge an object with another object and not remove nested null values', () => { - const result = utils.fastMerge(testObject, testObjectWithNullishValues, false, false, false); + const result = utils.fastMerge(testObject, testObjectWithNullishValues); expect(result.result).toEqual({ a: 'a', From 173802a9ef320f2e2ba0b8ac35081362aedfea1b Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 20 May 2025 13:18:29 +0200 Subject: [PATCH 26/58] add empty lines --- lib/utils.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/utils.ts b/lib/utils.ts index fbb6c3ab2..69bc63846 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -10,8 +10,10 @@ type FastMergeReplaceNullPatch = [string[], unknown]; type FastMergeOptions = { /** If true, null object values will be removed. */ shouldRemoveNestedNulls?: boolean; + /** If true, it means that we are batching merge changes before applying them to the Onyx value, so we must use a special logic to handle these changes. */ isBatchingMergeChanges?: boolean; + /** If true, any nested objects that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag will be completely replaced instead of merged. */ shouldReplaceMarkedObjects?: boolean; }; @@ -24,6 +26,7 @@ type FastMergeMetadata = { type FastMergeResult = { /** The result of the merge. */ result: TValue; + /** The path to the object that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag. */ replaceNullPatches: FastMergeReplaceNullPatch[]; }; From 351cc720cbd7c4b44edb6018533b464d2cbae688 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 20 May 2025 13:27:15 +0200 Subject: [PATCH 27/58] add missing continue --- lib/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/utils.ts b/lib/utils.ts index 69bc63846..289170cc3 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -131,6 +131,8 @@ function mergeObject>( if (!isMergeableObject(sourceProperty)) { // If source value is any other value we need to set the source value it directly. destination[key] = sourceProperty; + // eslint-disable-next-line no-continue + continue; } const targetProperty = targetObject?.[key]; From b7c84fbab543d29971af994adcd0d34d2d3ddf60 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Tue, 20 May 2025 16:02:54 +0200 Subject: [PATCH 28/58] fix: simplify fastMerge --- lib/OnyxUtils.ts | 2 +- lib/utils.ts | 70 +++++++++++++------------------------ tests/unit/fastMergeTest.ts | 2 +- 3 files changed, 26 insertions(+), 48 deletions(-) diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 98017bd33..c4c741873 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1268,7 +1268,7 @@ function mergeChanges | undefined, TChange ext // Object values are then merged one after the other return changes.reduce>( (modifiedData, change) => { - const fastMergeResult = utils.fastMerge(modifiedData.result, change, {isBatchingMergeChanges: true}); + const fastMergeResult = utils.fastMerge(modifiedData.result, change, {shouldMarkRemovedObjects: true}); // eslint-disable-next-line no-param-reassign modifiedData.result = fastMergeResult.result; // eslint-disable-next-line no-param-reassign diff --git a/lib/utils.ts b/lib/utils.ts index 289170cc3..90dab2eb6 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -12,7 +12,7 @@ type FastMergeOptions = { shouldRemoveNestedNulls?: boolean; /** If true, it means that we are batching merge changes before applying them to the Onyx value, so we must use a special logic to handle these changes. */ - isBatchingMergeChanges?: boolean; + shouldMarkRemovedObjects?: boolean; /** If true, any nested objects that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag will be completely replaced instead of merged. */ shouldReplaceMarkedObjects?: boolean; @@ -136,16 +136,33 @@ function mergeObject>( } const targetProperty = targetObject?.[key]; - const targetWithMarks = getTargetPropertyWithRemovalMark(targetProperty, sourceProperty, options, metadata, basePath); - const {finalDestinationProperty, stopTraversing} = replaceMarkedObjects(sourceProperty, options); + const targetPropertyWithMarks = (targetProperty ?? {}) as Record; + + // If we are batching merge changes and the previous merge change (targetValue) is null, + // it means we want to fully replace this object when merging the batched changes with the Onyx value. + // To achieve this, we first mark these nested objects with an internal flag. With the desired objects + // marked, when calling this method again with "shouldReplaceMarkedObjects" set to true we can proceed + // effectively replace them in the next condition. + if (options?.shouldMarkRemovedObjects && targetProperty === null) { + targetPropertyWithMarks[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; + metadata.replaceNullPatches.push([[...basePath], {...sourceProperty}]); + } + + // Later, when merging the batched changes with the Onyx value, if a nested object of the batched changes + // has the internal flag set, we replace the entire destination object with the source one and remove + // the flag. + if (options.shouldReplaceMarkedObjects && sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { + // We do a spread here in order to have a new object reference and allow us to delete the internal flag + // of the merged object only. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const {ONYX_INTERNALS__REPLACE_OBJECT_MARK: _mark, ...sourcePropertyWithoutMark} = sourceProperty; - if (stopTraversing) { - destination[key] = finalDestinationProperty; + destination[key] = sourcePropertyWithoutMark; // eslint-disable-next-line no-continue continue; } - destination[key] = fastMerge(targetWithMarks, sourceProperty, options, metadata, [...basePath, key]).result; + destination[key] = fastMerge(targetPropertyWithMarks, sourceProperty, options, metadata, [...basePath, key]).result; } return destination as TObject; @@ -166,51 +183,12 @@ function isMergeableObject>(value: unkno return isNonNullObject && !(value instanceof RegExp) && !(value instanceof Date) && !Array.isArray(value); } -function getTargetPropertyWithRemovalMark>( - targetProperty: unknown, - sourceProperty: Record, - options: FastMergeOptions, - metadata: FastMergeMetadata, - basePath: string[] = [], -): TObject { - const targetPropertyWithMarks = (targetProperty ?? {}) as Record; - - // If we are batching merge changes and the previous merge change (targetValue) is null, - // it means we want to fully replace this object when merging the batched changes with the Onyx value. - // To achieve this, we first mark these nested objects with an internal flag. With the desired objects - // marked, when calling this method again with "shouldReplaceMarkedObjects" set to true we can proceed - // effectively replace them in the next condition. - if (options?.isBatchingMergeChanges && targetProperty === null) { - targetPropertyWithMarks[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; - metadata.replaceNullPatches.push([[...basePath], {...sourceProperty}]); - } - - return targetPropertyWithMarks as TObject; -} - -function replaceMarkedObjects>(sourceProperty: TObject, options: FastMergeOptions): {finalDestinationProperty?: TObject; stopTraversing: boolean} { - // Then, when merging the batched changes with the Onyx value, if a nested object of the batched changes - // has the internal flag set, we replace the entire destination object with the source one and remove - // the flag. - if (options.shouldReplaceMarkedObjects && sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { - // We do a spread here in order to have a new object reference and allow us to delete the internal flag - // of the merged object only. - - const destinationProperty = {...sourceProperty}; - delete destinationProperty.ONYX_INTERNALS__REPLACE_OBJECT_MARK; - return {finalDestinationProperty: destinationProperty, stopTraversing: true}; - } - - // For the normal situations we'll just call `fastMerge()` again to merge the nested object. - return {stopTraversing: false}; -} - /** Deep removes the nested null values from the given value. */ function removeNestedNullValues | null>(value: TValue): TValue { if (typeof value === 'object' && !Array.isArray(value)) { return fastMerge(value, value, { shouldRemoveNestedNulls: true, - isBatchingMergeChanges: false, + shouldMarkRemovedObjects: false, shouldReplaceMarkedObjects: false, }).result; } diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index 4ed2cef6a..24016f62c 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -123,7 +123,7 @@ describe('fastMerge', () => { it('should add the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag to the target object when its source is set to null and "isBatchingMergeChanges" is true', () => { const result = utils.fastMerge(testMergeChanges[1], testMergeChanges[0], { shouldRemoveNestedNulls: true, - isBatchingMergeChanges: true, + shouldMarkRemovedObjects: true, }); expect(result.result).toEqual({ From ee5d461f6feaaecb2e8d49ab726b7c2a1608e91b Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 12:58:03 +0200 Subject: [PATCH 29/58] fix: further improve fastMerge code --- lib/Onyx.ts | 8 +- lib/OnyxCache.ts | 6 +- lib/OnyxUtils.ts | 30 +++++-- lib/storage/providers/IDBKeyValProvider.ts | 2 +- lib/storage/providers/MemoryOnlyProvider.ts | 2 +- lib/utils.ts | 96 +++++++++------------ tests/unit/fastMergeTest.ts | 6 +- tests/unit/onyxUtilsTest.ts | 6 +- 8 files changed, 82 insertions(+), 74 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 81c1da503..6fd5c0f93 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -307,7 +307,7 @@ function merge(key: TKey, changes: OnyxMergeInput): } try { - // We first only merge the changes, so we use OnyxUtils.batchMergeChanges() to combine all the changes into just one. + // We first only merge the changes, so we use OnyxUtils.mergeChanges() to combine all the changes into just one. const validChanges = mergeQueue[key].filter((change) => { const {isCompatible, existingValueType, newValueType} = utils.checkCompatibilityWithExistingValue(change, existingValue); if (!isCompatible) { @@ -332,7 +332,7 @@ function merge(key: TKey, changes: OnyxMergeInput): return Promise.resolve(); } - const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); + const {result: mergedValue} = OnyxUtils.mergeAndMarkChanges(validChanges, existingValue); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. const hasChanged = cache.hasValueChanged(key, mergedValue); @@ -757,7 +757,7 @@ function update(data: OnyxUpdate[]): Promise { // Remove the collection-related key from the updateQueue so that it won't be processed individually. delete updateQueue[key]; - const batchedChanges = OnyxUtils.mergeChanges(operations); + const batchedChanges = OnyxUtils.mergeAndMarkChanges(operations); if (operations[0] === null) { // eslint-disable-next-line no-param-reassign queue.set[key] = batchedChanges.result; @@ -790,7 +790,7 @@ function update(data: OnyxUpdate[]): Promise { Object.entries(updateQueue).forEach(([key, operations]) => { if (operations[0] === null) { - const batchedChanges = OnyxUtils.mergeChanges(operations).result; + const batchedChanges = OnyxUtils.mergeAndMarkChanges(operations).result; promises.push(() => set(key, batchedChanges)); return; } diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index d0d4585d4..adc52778b 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -167,7 +167,7 @@ class OnyxCache { this.storageMap = { ...utils.fastMerge(this.storageMap, data, { shouldRemoveNestedNulls: true, - shouldReplaceMarkedObjects: true, + objectRemovalMode: 'replace', }).result, }; @@ -232,6 +232,10 @@ class OnyxCache { const temp = []; while (numKeysToRemove > 0) { const value = iterator.next().value; + if (value === undefined) { + break; + } + temp.push(value); numKeysToRemove--; } diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index c4c741873..1003d3ae1 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -28,7 +28,7 @@ import type { OnyxValue, Selector, } from './types'; -import type {FastMergeResult} from './utils'; +import type {FastMergeOptions, FastMergeResult} from './utils'; import utils from './utils'; import type {WithOnyxState} from './withOnyx/types'; import type {DeferredTask} from './createDeferredTask'; @@ -1251,13 +1251,28 @@ function prepareKeyValuePairsForStorage(data: Record }, []); } +function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { + return applyMerge('merge', changes, existingValue); +} + +function mergeAndMarkChanges | undefined, TChange extends OnyxInput | undefined>( + changes: TChange[], + existingValue?: TValue, +): FastMergeResult { + return applyMerge('mark', changes, existingValue); +} + /** * Merges an array of changes with an existing value or creates a single change * * @param changes Array of changes that should be merged * @param existingValue The existing value that should be merged with the changes */ -function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { +function applyMerge | undefined, TChange extends OnyxInput | undefined>( + mode: 'merge' | 'mark', + changes: TChange[], + existingValue?: TValue, +): FastMergeResult { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { @@ -1268,11 +1283,14 @@ function mergeChanges | undefined, TChange ext // Object values are then merged one after the other return changes.reduce>( (modifiedData, change) => { - const fastMergeResult = utils.fastMerge(modifiedData.result, change, {shouldMarkRemovedObjects: true}); + const options: FastMergeOptions = mode === 'merge' ? {shouldRemoveNestedNulls: true, objectRemovalMode: 'replace'} : {objectRemovalMode: 'mark'}; + const {result, replaceNullPatches} = utils.fastMerge(modifiedData.result, change, options); + // eslint-disable-next-line no-param-reassign - modifiedData.result = fastMergeResult.result; + modifiedData.result = result; // eslint-disable-next-line no-param-reassign - modifiedData.replaceNullPatches = [...modifiedData.replaceNullPatches, ...fastMergeResult.replaceNullPatches]; + modifiedData.replaceNullPatches = [...modifiedData.replaceNullPatches, ...replaceNullPatches]; + return modifiedData; }, { @@ -1296,7 +1314,6 @@ function initializeWithDefaultKeyStates(): Promise { const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, { shouldRemoveNestedNulls: true, - shouldReplaceMarkedObjects: true, }).result; cache.merge(merged ?? {}); @@ -1488,6 +1505,7 @@ const OnyxUtils = { removeNullValues, prepareKeyValuePairsForStorage, mergeChanges, + mergeAndMarkChanges, initializeWithDefaultKeyStates, getSnapshotKey, multiGet, diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index 036592d7b..c61fc851e 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -51,7 +51,7 @@ const provider: StorageProvider = { const prev = values[index]; const newValue = utils.fastMerge(prev as Record, value as Record, { shouldRemoveNestedNulls: true, - shouldReplaceMarkedObjects: true, + objectRemovalMode: 'replace', }).result; return promisifyRequest(store.put(newValue, key)); }); diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index a6016f99c..1af95a6e3 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -88,7 +88,7 @@ const provider: StorageProvider = { const existingValue = store[key] as Record; const newValue = utils.fastMerge(existingValue, value as Record, { shouldRemoveNestedNulls: true, - shouldReplaceMarkedObjects: true, + objectRemovalMode: 'replace', }).result as OnyxValue; set(key, newValue); diff --git a/lib/utils.ts b/lib/utils.ts index 90dab2eb6..16b83a2eb 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -11,8 +11,12 @@ type FastMergeOptions = { /** If true, null object values will be removed. */ shouldRemoveNestedNulls?: boolean; - /** If true, it means that we are batching merge changes before applying them to the Onyx value, so we must use a special logic to handle these changes. */ - shouldMarkRemovedObjects?: boolean; + /** + * If set to "mark", we will mark objects that are set to null instead of simply removing them, + * so that we can batch changes together, without loosing information about the object removal. + * If set to "replace", we will completely replace the marked objects with the new value instead of merging them. + * */ + objectRemovalMode?: 'mark' | 'replace' | 'none'; /** If true, any nested objects that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag will be completely replaced instead of merged. */ shouldReplaceMarkedObjects?: boolean; @@ -53,11 +57,9 @@ function fastMerge(target: TValue, source: TValue, options?: FastMergeOp return {result: source, replaceNullPatches: metadata.replaceNullPatches}; } - const optionsWithDefaults = { - shouldRemoveNestedNulls: false, - isBatchingMergeChanges: false, - shouldReplaceMarkedObjects: false, - ...options, + const optionsWithDefaults: FastMergeOptions = { + shouldRemoveNestedNulls: options?.shouldRemoveNestedNulls ?? false, + objectRemovalMode: options?.objectRemovalMode ?? 'none', }; const mergedValue = mergeObject(target, source as Record, optionsWithDefaults, metadata, basePath) as TValue; @@ -90,80 +92,65 @@ function mergeObject>( // If "shouldRemoveNestedNulls" is true, we want to remove null values from the merged object // and therefore we need to omit keys where either the source or target value is null. if (targetObject) { - // eslint-disable-next-line no-restricted-syntax, guard-for-in - for (const key in targetObject) { + Object.keys(targetObject).forEach((key) => { const targetProperty = targetObject?.[key]; - if (targetProperty === undefined) { - // eslint-disable-next-line no-continue - continue; - } + const sourceProperty = source?.[key]; // If "shouldRemoveNestedNulls" is true, we want to remove (nested) null values from the merged object. // If either the source or target value is null, we want to omit the key from the merged object. - const sourceProperty = source?.[key]; - const isSourceOrTargetNull = targetProperty === null || sourceProperty === null; - const shouldOmitTargetKey = options.shouldRemoveNestedNulls && isSourceOrTargetNull; + const shouldOmitNullishProperty = options.shouldRemoveNestedNulls && (targetProperty === null || sourceProperty === null); - if (!shouldOmitTargetKey) { - destination[key] = targetProperty; + if (targetProperty === undefined || shouldOmitNullishProperty) { + return; } - } + + destination[key] = targetProperty; + }); } // After copying over all keys from the target object, we want to merge the source object into the destination object. - // eslint-disable-next-line no-restricted-syntax, guard-for-in - for (const key in source) { + Object.keys(source).forEach((key) => { + let targetProperty = targetObject?.[key]; const sourceProperty = source?.[key] as Record; - if (sourceProperty === undefined) { - // eslint-disable-next-line no-continue - continue; - } - // If "shouldRemoveNestedNulls" is set to true and the source value is null, - // we don't want to set/merge the source value into the merged object. - const shouldOmitSourceKey = options.shouldRemoveNestedNulls && sourceProperty === null; - if (shouldOmitSourceKey) { - // eslint-disable-next-line no-continue - continue; + // If "shouldRemoveNestedNulls" is true, we want to remove (nested) null values from the merged object. + // If either the source value is null, we want to omit the key from the merged object. + const shouldOmitNullishProperty = options.shouldRemoveNestedNulls && sourceProperty === null; + + if (sourceProperty === undefined || shouldOmitNullishProperty) { + return; } - // If the source value is a mergable object, we want to merge it into the target value. + // If source value is not a mergable object, we need to set the source value it directly. if (!isMergeableObject(sourceProperty)) { - // If source value is any other value we need to set the source value it directly. destination[key] = sourceProperty; - // eslint-disable-next-line no-continue - continue; + return; } - const targetProperty = targetObject?.[key]; - const targetPropertyWithMarks = (targetProperty ?? {}) as Record; - - // If we are batching merge changes and the previous merge change (targetValue) is null, + // If "shouldMarkRemovedObjects" is enabled and the previous merge change (targetProperty) is null, // it means we want to fully replace this object when merging the batched changes with the Onyx value. - // To achieve this, we first mark these nested objects with an internal flag. With the desired objects - // marked, when calling this method again with "shouldReplaceMarkedObjects" set to true we can proceed - // effectively replace them in the next condition. - if (options?.shouldMarkRemovedObjects && targetProperty === null) { - targetPropertyWithMarks[ONYX_INTERNALS__REPLACE_OBJECT_MARK] = true; + // To achieve this, we first mark these nested objects with an internal flag. + // When calling fastMerge again with "shouldReplaceMarkedObjects" enabled, the marked objects will be removed. + if (options.objectRemovalMode === 'mark' && targetProperty === null) { + targetProperty = {[ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true}; metadata.replaceNullPatches.push([[...basePath], {...sourceProperty}]); } // Later, when merging the batched changes with the Onyx value, if a nested object of the batched changes // has the internal flag set, we replace the entire destination object with the source one and remove // the flag. - if (options.shouldReplaceMarkedObjects && sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { + if (options.objectRemovalMode === 'replace' && sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { // We do a spread here in order to have a new object reference and allow us to delete the internal flag // of the merged object only. // eslint-disable-next-line @typescript-eslint/no-unused-vars - const {ONYX_INTERNALS__REPLACE_OBJECT_MARK: _mark, ...sourcePropertyWithoutMark} = sourceProperty; - - destination[key] = sourcePropertyWithoutMark; - // eslint-disable-next-line no-continue - continue; + delete sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]; + // const {ONYX_INTERNALS__REPLACE_OBJECT_MARK: _mark, ...sourcePropertyWithoutMark} = sourceProperty; + destination[key] = sourceProperty; + return; } - destination[key] = fastMerge(targetPropertyWithMarks, sourceProperty, options, metadata, [...basePath, key]).result; - } + destination[key] = fastMerge(targetProperty, sourceProperty, options, metadata, [...basePath, key]).result; + }); return destination as TObject; } @@ -188,8 +175,7 @@ function removeNestedNullValues | null>(value: if (typeof value === 'object' && !Array.isArray(value)) { return fastMerge(value, value, { shouldRemoveNestedNulls: true, - shouldMarkRemovedObjects: false, - shouldReplaceMarkedObjects: false, + objectRemovalMode: 'replace', }).result; } @@ -294,4 +280,4 @@ export default { hasWithOnyxInstance, ONYX_INTERNALS__REPLACE_OBJECT_MARK, }; -export type {FastMergeResult, FastMergeReplaceNullPatch}; +export type {FastMergeResult, FastMergeReplaceNullPatch, FastMergeOptions}; diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index 24016f62c..8821bdb1c 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -120,10 +120,10 @@ describe('fastMerge', () => { expect(result.result).toEqual(testObject); }); - it('should add the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag to the target object when its source is set to null and "isBatchingMergeChanges" is true', () => { + it('should add the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag to the merged object when the change is set to null and "objectRemovalMode" is set to "mark"', () => { const result = utils.fastMerge(testMergeChanges[1], testMergeChanges[0], { shouldRemoveNestedNulls: true, - shouldMarkRemovedObjects: true, + objectRemovalMode: 'mark', }); expect(result.result).toEqual({ @@ -152,7 +152,7 @@ describe('fastMerge', () => { }, { shouldRemoveNestedNulls: true, - shouldReplaceMarkedObjects: true, + objectRemovalMode: 'replace', }, ); diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index c017ef3d8..330637df8 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -151,7 +151,7 @@ describe('OnyxUtils', () => { }); }); - describe('mergeChanges', () => { + describe('mergeAndMarkChanges', () => { it("should return the last change if it's an array", () => { const result = OnyxUtils.mergeChanges([...testMergeChanges, [0, 1, 2]], testObject); @@ -199,7 +199,7 @@ describe('OnyxUtils', () => { }); it('should apply the replacement markers if the we have properties with objects being removed and added back during the changes', () => { - const result = OnyxUtils.mergeChanges(testMergeChanges); + const result = OnyxUtils.mergeAndMarkChanges(testMergeChanges); expect(result.result).toEqual({ b: { @@ -223,7 +223,7 @@ describe('OnyxUtils', () => { }); it('should 2', () => { - const result = OnyxUtils.mergeChanges([ + const result = OnyxUtils.mergeAndMarkChanges([ { // Removing the "originalMessage" object in this update. // Any subsequent changes to this object should completely replace the existing object in store. From 6d6fbb460bb4b9c71d099d3a94d2e95066824820 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 15:16:20 +0200 Subject: [PATCH 30/58] remove old comment --- lib/Onyx.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 6fd5c0f93..419898c89 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -307,7 +307,6 @@ function merge(key: TKey, changes: OnyxMergeInput): } try { - // We first only merge the changes, so we use OnyxUtils.mergeChanges() to combine all the changes into just one. const validChanges = mergeQueue[key].filter((change) => { const {isCompatible, existingValueType, newValueType} = utils.checkCompatibilityWithExistingValue(change, existingValue); if (!isCompatible) { From 24b226962289e59ee7ea4b6ba699483a5a536c50 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 16:15:06 +0200 Subject: [PATCH 31/58] fix: missing key in path --- lib/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils.ts b/lib/utils.ts index 16b83a2eb..9a282cf12 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -133,7 +133,7 @@ function mergeObject>( // When calling fastMerge again with "shouldReplaceMarkedObjects" enabled, the marked objects will be removed. if (options.objectRemovalMode === 'mark' && targetProperty === null) { targetProperty = {[ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true}; - metadata.replaceNullPatches.push([[...basePath], {...sourceProperty}]); + metadata.replaceNullPatches.push([[...basePath, key], {...sourceProperty}]); } // Later, when merging the batched changes with the Onyx value, if a nested object of the batched changes From 4c23dba4ea446fb0fc13acaece2c93d02b73f78b Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 16:15:22 +0200 Subject: [PATCH 32/58] fix: onyxUtilsTest --- tests/unit/onyxUtilsTest.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index 330637df8..2e065fe71 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -151,15 +151,15 @@ describe('OnyxUtils', () => { }); }); - describe('mergeAndMarkChanges', () => { + describe('mergeChanges', () => { it("should return the last change if it's an array", () => { - const result = OnyxUtils.mergeChanges([...testMergeChanges, [0, 1, 2]], testObject); + const {result} = OnyxUtils.mergeAndMarkChanges([...testMergeChanges, [0, 1, 2]], testObject); expect(result).toEqual([0, 1, 2]); }); it("should return the last change if the changes aren't objects", () => { - const result = OnyxUtils.mergeChanges(['a', 0, 'b', 1], testObject); + const {result} = OnyxUtils.mergeChanges(['a', 0, 'b', 1], testObject); expect(result).toEqual(1); }); @@ -180,7 +180,7 @@ describe('OnyxUtils', () => { }, }; - const result = OnyxUtils.mergeChanges([batchedChanges], testObject); + const {result} = OnyxUtils.mergeChanges([batchedChanges], testObject); expect(result).toEqual({ a: 'a', @@ -197,11 +197,13 @@ describe('OnyxUtils', () => { }, }); }); + }); + describe('mergeAndMarkChanges', () => { it('should apply the replacement markers if the we have properties with objects being removed and added back during the changes', () => { - const result = OnyxUtils.mergeAndMarkChanges(testMergeChanges); + const {result, replaceNullPatches} = OnyxUtils.mergeAndMarkChanges(testMergeChanges); - expect(result.result).toEqual({ + expect(result).toEqual({ b: { d: { i: 'i', @@ -215,7 +217,7 @@ describe('OnyxUtils', () => { }, }, }); - expect(result.replaceNullPatches).toEqual([ + expect(replaceNullPatches).toEqual([ [['b', 'd'], {i: 'i'}], [['b', 'd'], {i: 'i', j: 'j'}], [['b', 'g'], {k: 'k'}], @@ -223,7 +225,7 @@ describe('OnyxUtils', () => { }); it('should 2', () => { - const result = OnyxUtils.mergeAndMarkChanges([ + const {result, replaceNullPatches} = OnyxUtils.mergeAndMarkChanges([ { // Removing the "originalMessage" object in this update. // Any subsequent changes to this object should completely replace the existing object in store. @@ -252,7 +254,7 @@ describe('OnyxUtils', () => { }, ]); - expect(result.result).toEqual({ + expect(result).toEqual({ originalMessage: { errorMessage: 'newErrorMessage', [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, @@ -266,7 +268,8 @@ describe('OnyxUtils', () => { }, }, }); - expect(result.replaceNullPatches).toEqual([ + + expect(replaceNullPatches).toEqual([ [['originalMessage'], {errorMessage: 'newErrorMessage'}], [['receipt', 'nestedObject'], {nestedKey2: 'newNestedKey2'}], ]); From 07447249b69ba8cac169a18419fa08fd1a056b58 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 16:22:23 +0200 Subject: [PATCH 33/58] fix: tests --- lib/Onyx.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 419898c89..51e655ac9 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -331,7 +331,7 @@ function merge(key: TKey, changes: OnyxMergeInput): return Promise.resolve(); } - const {result: mergedValue} = OnyxUtils.mergeAndMarkChanges(validChanges, existingValue); + const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. const hasChanged = cache.hasValueChanged(key, mergedValue); @@ -427,10 +427,12 @@ function mergeCollection( const existingKeyCollection = existingKeys.reduce((obj: OnyxInputKeyValueMapping, key) => { const {isCompatible, existingValueType, newValueType} = utils.checkCompatibilityWithExistingValue(resultCollection[key], cachedCollectionForExistingKeys[key]); + if (!isCompatible) { Logger.logAlert(logMessages.incompatibleUpdateAlert(key, 'mergeCollection', existingValueType, newValueType)); return obj; } + // eslint-disable-next-line no-param-reassign obj[key] = resultCollection[key]; return obj; From ccedbd8bb96d6b484e9211f020756a591f7a26db Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 16:52:26 +0200 Subject: [PATCH 34/58] further simplify code --- API-INTERNAL.md | 13 ----- lib/Onyx.ts | 40 +++++++-------- lib/OnyxUtils.ts | 56 ++++++++------------- lib/storage/InstanceSync/index.web.ts | 8 +-- lib/storage/providers/IDBKeyValProvider.ts | 2 +- lib/storage/providers/MemoryOnlyProvider.ts | 4 +- lib/storage/providers/SQLiteProvider.ts | 23 +++++---- lib/storage/providers/types.ts | 22 ++++---- lib/types.ts | 5 +- 9 files changed, 71 insertions(+), 102 deletions(-) diff --git a/API-INTERNAL.md b/API-INTERNAL.md index a098586b3..ab9dd07cf 100644 --- a/API-INTERNAL.md +++ b/API-INTERNAL.md @@ -139,11 +139,6 @@ whatever it is we attempted to do.

broadcastUpdate()

Notifies subscribers and writes current value to cache

-
removeNullValues() ⇒
-

Removes a key from storage if the value is null. -Otherwise removes all nested null values in objects, -if shouldRemoveNestedNulls is true and returns the object.

-
prepareKeyValuePairsForStorage() ⇒

Storage expects array like: [["@MyApp_user", value_1], ["@MyApp_key", value_2]] This method transforms an object like {'@MyApp_user': myUserValue, '@MyApp_key': myKeyValue} @@ -464,14 +459,6 @@ whatever it is we attempted to do. ## broadcastUpdate() Notifies subscribers and writes current value to cache -**Kind**: global function - - -## removeNullValues() ⇒ -Removes a key from storage if the value is null. -Otherwise removes all nested null values in objects, -if shouldRemoveNestedNulls is true and returns the object. - **Kind**: global function **Returns**: The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 51e655ac9..3039b611a 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -27,6 +27,7 @@ import type { OnyxValue, OnyxInput, OnyxMethodMap, + MultiMergeReplaceNullPatches, } from './types'; import OnyxUtils from './OnyxUtils'; import logMessages from './logMessages'; @@ -169,38 +170,31 @@ function set(key: TKey, value: OnyxSetInput): Promis return Promise.resolve(); } - // If the value is null, we remove the key from storage - const {value: valueAfterRemoving, wasRemoved} = OnyxUtils.removeNullValues(key, value); - - const logSetCall = (hasChanged = true) => { - // Logging properties only since values could be sensitive things we don't want to log - Logger.logInfo(`set called for key: ${key}${_.isObject(value) ? ` properties: ${_.keys(value).join(',')}` : ''} hasChanged: ${hasChanged}`); - }; - - // Calling "OnyxUtils.removeNullValues" removes the key from storage and cache and updates the subscriber. + // If the change is null, we can just delete the key. // Therefore, we don't need to further broadcast and update the value so we can return early. - if (wasRemoved) { - logSetCall(); + if (value === null) { + OnyxUtils.remove(key); + Logger.logInfo(`set called for key: ${key} => null passed, so key was removed`); return Promise.resolve(); } - const valueWithoutNullValues = valueAfterRemoving as OnyxValue; - const hasChanged = cache.hasValueChanged(key, valueWithoutNullValues); + const valueWithoutNestedNullValues = utils.removeNestedNullValues(value) as OnyxValue; + const hasChanged = cache.hasValueChanged(key, valueWithoutNestedNullValues); - logSetCall(hasChanged); + Logger.logInfo(`set called for key: ${key}${_.isObject(value) ? ` properties: ${_.keys(value).join(',')}` : ''} hasChanged: ${hasChanged}`); // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. - const updatePromise = OnyxUtils.broadcastUpdate(key, valueWithoutNullValues, hasChanged); + const updatePromise = OnyxUtils.broadcastUpdate(key, valueWithoutNestedNullValues, hasChanged); // If the value has not changed or the key got removed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. if (!hasChanged) { return updatePromise; } - return Storage.setItem(key, valueWithoutNullValues) - .catch((error) => OnyxUtils.evictStorageAndRetry(error, set, key, valueWithoutNullValues)) + return Storage.setItem(key, valueWithoutNestedNullValues) + .catch((error) => OnyxUtils.evictStorageAndRetry(error, set, key, valueWithoutNestedNullValues)) .then(() => { - OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.SET, key, valueWithoutNullValues); + OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.SET, key, valueWithoutNestedNullValues); return updatePromise; }); } @@ -323,10 +317,10 @@ function merge(key: TKey, changes: OnyxMergeInput): delete mergeQueue[key]; delete mergeQueuePromise[key]; - // Calling "OnyxUtils.remove" removes the key from storage and cache and updates the subscriber. + // If the last change is null, we can just delete the key. // Therefore, we don't need to further broadcast and update the value so we can return early. if (validChanges.at(-1) === null) { - Logger.logInfo(`merge called for key: ${key} was removed`); + Logger.logInfo(`merge called for key: ${key} => null passed, so key was removed`); OnyxUtils.remove(key); return Promise.resolve(); } @@ -376,7 +370,7 @@ function merge(key: TKey, changes: OnyxMergeInput): function mergeCollection( collectionKey: TKey, collection: OnyxMergeCollectionInput, - mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches'], + mergeReplaceNullPatches?: MultiMergeReplaceNullPatches, ): Promise { if (!OnyxUtils.isValidNonEmptyCollectionForMerge(collection)) { Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.'); @@ -449,7 +443,7 @@ function mergeCollection( // When (multi-)merging the values with the existing values in storage, // we don't want to remove nested null values from the data that we pass to the storage layer, // because the storage layer uses them to remove nested keys from storage natively. - const keyValuePairsForExistingCollection = OnyxUtils.prepareKeyValuePairsForStorage(existingKeyCollection, false); + const keyValuePairsForExistingCollection = OnyxUtils.prepareKeyValuePairsForStorage(existingKeyCollection, false, mergeReplaceNullPatches); // We can safely remove nested null values when using (multi-)set, // because we will simply overwrite the existing values in storage. @@ -464,7 +458,7 @@ function mergeCollection( // New keys will be added via multiSet while existing keys will be updated using multiMerge // This is because setting a key that doesn't exist yet with multiMerge will throw errors if (keyValuePairsForExistingCollection.length > 0) { - promises.push(Storage.multiMerge(keyValuePairsForExistingCollection, mergeReplaceNullPatches)); + promises.push(Storage.multiMerge(keyValuePairsForExistingCollection)); } if (keyValuePairsForNewCollection.length > 0) { diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 1003d3ae1..e89769404 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -20,6 +20,7 @@ import type { DefaultConnectOptions, KeyValueMapping, Mapping, + MultiMergeReplaceNullPatches, OnyxCollection, OnyxEntry, OnyxInput, @@ -35,6 +36,7 @@ import type {DeferredTask} from './createDeferredTask'; import createDeferredTask from './createDeferredTask'; import * as GlobalSettings from './GlobalSettings'; import decorateWithMetrics from './metrics'; +import type {StorageKeyValuePair} from './storage/providers/types'; // Method constants const METHOD = { @@ -1204,34 +1206,6 @@ function hasPendingMergeForKey(key: OnyxKey): boolean { return !!mergeQueue[key]; } -type RemoveNullValuesOutput | undefined> = { - value: Value; - wasRemoved: boolean; -}; - -/** - * Removes a key from storage if the value is null. - * Otherwise removes all nested null values in objects, - * if shouldRemoveNestedNulls is true and returns the object. - * - * @returns The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely - */ -function removeNullValues | undefined>(key: OnyxKey, value: Value, shouldRemoveNestedNulls = true): RemoveNullValuesOutput { - if (value === null) { - remove(key); - return {value, wasRemoved: true}; - } - - if (value === undefined) { - return {value, wasRemoved: false}; - } - - // We can remove all null values in an object by merging it with itself - // utils.fastMerge recursively goes through the object and removes all null values - // Passing two identical objects as source and target to fastMerge will not change it, but only remove the null values - return {value: shouldRemoveNestedNulls ? utils.removeNestedNullValues(value) : value, wasRemoved: false}; -} - /** * Storage expects array like: [["@MyApp_user", value_1], ["@MyApp_key", value_2]] * This method transforms an object like {'@MyApp_user': myUserValue, '@MyApp_key': myKeyValue} @@ -1239,16 +1213,27 @@ function removeNullValues | undefined>(key: Ony * @return an array of key - value pairs <[key, value]> */ -function prepareKeyValuePairsForStorage(data: Record>, shouldRemoveNestedNulls: boolean): Array<[OnyxKey, OnyxInput]> { - return Object.entries(data).reduce]>>((pairs, [key, value]) => { - const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value, shouldRemoveNestedNulls); +function prepareKeyValuePairsForStorage( + data: Record>, + shouldRemoveNestedNulls?: boolean, + replaceNullPatches?: MultiMergeReplaceNullPatches, +): StorageKeyValuePair[] { + const pairs: StorageKeyValuePair[] = []; + + Object.entries(data).forEach(([key, value]) => { + if (value === null) { + remove(key); + return; + } + + const valueWithoutNestedNullValues = shouldRemoveNestedNulls ?? true ? utils.removeNestedNullValues(value) : value; - if (!wasRemoved && valueAfterRemoving !== undefined) { - pairs.push([key, valueAfterRemoving]); + if (valueWithoutNestedNullValues !== undefined) { + pairs.push([key, valueWithoutNestedNullValues, replaceNullPatches?.[key]]); } + }); - return pairs; - }, []); + return pairs; } function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { @@ -1502,7 +1487,6 @@ const OnyxUtils = { evictStorageAndRetry, broadcastUpdate, hasPendingMergeForKey, - removeNullValues, prepareKeyValuePairsForStorage, mergeChanges, mergeAndMarkChanges, diff --git a/lib/storage/InstanceSync/index.web.ts b/lib/storage/InstanceSync/index.web.ts index 67b309791..99a7fe325 100644 --- a/lib/storage/InstanceSync/index.web.ts +++ b/lib/storage/InstanceSync/index.web.ts @@ -5,7 +5,7 @@ */ import type {OnyxKey} from '../../types'; import NoopProvider from '../providers/NoopProvider'; -import type {KeyList, OnStorageKeyChanged} from '../providers/types'; +import type {StorageKeyList, OnStorageKeyChanged} from '../providers/types'; import type StorageProvider from '../providers/types'; const SYNC_ONYX = 'SYNC_ONYX'; @@ -19,7 +19,7 @@ function raiseStorageSyncEvent(onyxKey: OnyxKey) { global.localStorage.removeItem(SYNC_ONYX); } -function raiseStorageSyncManyKeysEvent(onyxKeys: KeyList) { +function raiseStorageSyncManyKeysEvent(onyxKeys: StorageKeyList) { onyxKeys.forEach((onyxKey) => { raiseStorageSyncEvent(onyxKey); }); @@ -54,12 +54,12 @@ const InstanceSync = { multiSet: raiseStorageSyncManyKeysEvent, mergeItem: raiseStorageSyncEvent, clear: (clearImplementation: () => void) => { - let allKeys: KeyList; + let allKeys: StorageKeyList; // The keys must be retrieved before storage is cleared or else the list of keys would be empty return storage .getAllKeys() - .then((keys: KeyList) => { + .then((keys: StorageKeyList) => { allKeys = keys; }) .then(() => clearImplementation()) diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index c61fc851e..64d419a83 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -70,7 +70,7 @@ const provider: StorageProvider = { } return true; - }); + }) as Array<[IDBValidKey, unknown]>; return setMany(pairsWithoutNull, idbKeyValStore); }, diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index 1af95a6e3..1510d20cb 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -1,7 +1,7 @@ import _ from 'underscore'; import utils from '../../utils'; import type StorageProvider from './types'; -import type {KeyValuePair} from './types'; +import type {StorageKeyValuePair} from './types'; import type {OnyxKey, OnyxValue} from '../../types'; type Store = Record>; @@ -49,7 +49,7 @@ const provider: StorageProvider = { new Promise((resolve) => { this.getItem(key).then((value) => resolve([key, value])); }), - ) as Array>; + ) as Array>; return Promise.all(getPromises); }, diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index fd3363f79..bdcde2b76 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -8,7 +8,7 @@ import {open} from 'react-native-quick-sqlite'; import type {FastMergeReplaceNullPatch} from '../../utils'; import utils from '../../utils'; import type StorageProvider from './types'; -import type {KeyList, KeyValuePairList} from './types'; +import type {StorageKeyList, StorageKeyValuePair} from './types'; const DB_NAME = 'OnyxDB'; let db: QuickSQLiteConnection; @@ -61,7 +61,7 @@ const provider: StorageProvider = { return db.executeAsync(command, keys).then(({rows}) => { // eslint-disable-next-line no-underscore-dangle const result = rows?._array.map((row) => [row.record_key, JSON.parse(row.valueJSON)]); - return (result ?? []) as KeyValuePairList; + return (result ?? []) as StorageKeyValuePair[]; }); }, setItem(key, value) { @@ -74,7 +74,7 @@ const provider: StorageProvider = { } return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]).then(() => undefined); }, - multiMerge(pairs, mergeReplaceNullPatches) { + multiMerge(pairs) { const commands: SQLBatchTuple[] = []; const patchQuery = `INSERT INTO keyvaluepairs (record_key, valueJSON) @@ -93,13 +93,14 @@ const provider: StorageProvider = { // eslint-disable-next-line @typescript-eslint/prefer-for-of for (let i = 0; i < nonNullishPairs.length; i++) { - const pair = nonNullishPairs[i]; - const value = JSON.stringify(pair[1], replacer); - patchQueryArguments.push([pair[0], value]); + const [key, value, replaceNullPatches] = nonNullishPairs[i]; - const patches = mergeReplaceNullPatches?.[pair[0]] ?? []; + const valueAfterReplace = JSON.stringify(value, replacer); + patchQueryArguments.push([key, valueAfterReplace]); + + const patches = replaceNullPatches ?? []; if (patches.length > 0) { - const queries = generateJSONReplaceSQLQueries(pair[0], patches); + const queries = generateJSONReplaceSQLQueries(key, patches); if (queries.length > 0) { replaceQueryArguments.push(...queries); @@ -114,15 +115,15 @@ const provider: StorageProvider = { return db.executeBatchAsync(commands).then(() => undefined); }, - mergeItem(key, change) { + mergeItem(key, change, replaceNullPatches) { // Since Onyx already merged the existing value with the changes, we can just set the value directly. - return this.multiMerge([[key, change]]); + return this.multiMerge([[key, change, replaceNullPatches]]); }, getAllKeys: () => db.executeAsync('SELECT record_key FROM keyvaluepairs;').then(({rows}) => { // eslint-disable-next-line no-underscore-dangle const result = rows?._array.map((row) => row.record_key); - return (result ?? []) as KeyList; + return (result ?? []) as StorageKeyList; }), removeItem: (key) => db.executeAsync('DELETE FROM keyvaluepairs WHERE record_key = ?;', [key]).then(() => undefined), removeItems: (keys) => { diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index fc3b3cade..db7525aa5 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -1,8 +1,8 @@ -import type {MixedOperationsQueue, OnyxKey, OnyxValue} from '../../types'; +import type {OnyxKey, OnyxValue} from '../../types'; +import type {FastMergeReplaceNullPatch} from '../../utils'; -type KeyValuePair = [OnyxKey, OnyxValue]; -type KeyList = OnyxKey[]; -type KeyValuePairList = KeyValuePair[]; +type StorageKeyValuePair = [key: OnyxKey, value: OnyxValue, replaceNullPatches?: FastMergeReplaceNullPatch[]]; +type StorageKeyList = OnyxKey[]; type DatabaseSize = { bytesUsed: number; @@ -28,7 +28,7 @@ type StorageProvider = { /** * Get multiple key-value pairs for the given array of keys in a batch */ - multiGet: (keys: KeyList) => Promise; + multiGet: (keys: StorageKeyList) => Promise; /** * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string @@ -38,23 +38,23 @@ type StorageProvider = { /** * Stores multiple key-value pairs in a batch */ - multiSet: (pairs: KeyValuePairList) => Promise; + multiSet: (pairs: StorageKeyValuePair[]) => Promise; /** * Multiple merging of existing and new values in a batch */ - multiMerge: (pairs: KeyValuePairList, mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches']) => Promise; + multiMerge: (pairs: StorageKeyValuePair[]) => Promise; /** * Merges an existing value with a new one * @param change - the change to merge with the existing value */ - mergeItem: (key: TKey, change: OnyxValue) => Promise; + mergeItem: (key: TKey, change: OnyxValue, replaceNullPatches?: FastMergeReplaceNullPatch[]) => Promise; /** * Returns all keys available in storage */ - getAllKeys: () => Promise; + getAllKeys: () => Promise; /** * Removes given key and its value from storage @@ -64,7 +64,7 @@ type StorageProvider = { /** * Removes given keys and their values from storage */ - removeItems: (keys: KeyList) => Promise; + removeItems: (keys: StorageKeyList) => Promise; /** * Clears absolutely everything from storage @@ -83,4 +83,4 @@ type StorageProvider = { }; export default StorageProvider; -export type {KeyList, KeyValuePair, KeyValuePairList, OnStorageKeyChanged}; +export type {StorageKeyList, StorageKeyValuePair, OnStorageKeyChanged}; diff --git a/lib/types.ts b/lib/types.ts index cd5556b89..7dbdb1ca7 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -486,12 +486,14 @@ type InitOptions = { // eslint-disable-next-line @typescript-eslint/no-explicit-any type GenericFunction = (...args: any[]) => any; +type MultiMergeReplaceNullPatches = {[TKey in OnyxKey]: FastMergeReplaceNullPatch[]}; + /** * Represents a combination of Merge and Set operations that should be executed in Onyx */ type MixedOperationsQueue = { merge: OnyxInputKeyValueMapping; - mergeReplaceNullPatches: {[TKey in OnyxKey]: FastMergeReplaceNullPatch[]}; + mergeReplaceNullPatches: MultiMergeReplaceNullPatches; set: OnyxInputKeyValueMapping; }; @@ -533,5 +535,6 @@ export type { OnyxValue, Selector, WithOnyxConnectOptions, + MultiMergeReplaceNullPatches, MixedOperationsQueue, }; From 8391cc7628ff629a513af4550bec0a01cc78d55f Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 17:19:24 +0200 Subject: [PATCH 35/58] keep performance merge logic on native --- lib/Onyx.ts | 23 ++-------------------- lib/OnyxMerge.native.ts | 43 +++++++++++++++++++++++++++++++++++++++++ lib/OnyxMerge.ts | 35 +++++++++++++++++++++++++++++++++ lib/OnyxUtils.ts | 19 ++++++------------ 4 files changed, 86 insertions(+), 34 deletions(-) create mode 100644 lib/OnyxMerge.native.ts create mode 100644 lib/OnyxMerge.ts diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 3039b611a..5bee462a1 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-continue */ import _ from 'underscore'; import lodashPick from 'lodash/pick'; import * as Logger from './Logger'; @@ -35,6 +34,7 @@ import type {Connection} from './OnyxConnectionManager'; import connectionManager from './OnyxConnectionManager'; import * as GlobalSettings from './GlobalSettings'; import decorateWithMetrics from './metrics'; +import OnyxMerge from './OnyxMerge.native'; /** Initialize the store with actions and listening for storage events */ function init({ @@ -325,26 +325,7 @@ function merge(key: TKey, changes: OnyxMergeInput): return Promise.resolve(); } - const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); - - // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. - const hasChanged = cache.hasValueChanged(key, mergedValue); - - // Logging properties only since values could be sensitive things we don't want to log. - Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`); - - // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. - const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue, hasChanged); - - // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. - if (!hasChanged) { - return updatePromise; - } - - return Storage.setItem(key, mergedValue as OnyxValue).then(() => { - OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, mergedValue); - return updatePromise; - }); + return OnyxMerge.applyMerge(key, existingValue, validChanges); } catch (error) { Logger.logAlert(`An error occurred while applying merge for key: ${key}, Error: ${error}`); return Promise.resolve(); diff --git a/lib/OnyxMerge.native.ts b/lib/OnyxMerge.native.ts new file mode 100644 index 000000000..f18589e77 --- /dev/null +++ b/lib/OnyxMerge.native.ts @@ -0,0 +1,43 @@ +import _ from 'underscore'; +import * as Logger from './Logger'; +import OnyxUtils from './OnyxUtils'; +import type {OnyxKey, OnyxValue} from './types'; +import cache from './OnyxCache'; +import Storage from './storage'; + +function applyMerge(key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise { + // If any of the changes is null, we need to discard the existing value. + const baseValue = validChanges.includes(null) ? undefined : existingValue; + + // We first batch the changes into a single change with object removal marks, + // so that SQLite can merge the changes more efficiently. + const {result: batchedChanges, replaceNullPatches} = OnyxUtils.mergeAndMarkChanges(validChanges); + + // We then merge the batched changes with the existing value, because we need to final merged value to broadcast to subscribers. + const {result: mergedValue} = OnyxUtils.mergeChanges([batchedChanges], baseValue); + + // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. + const hasChanged = cache.hasValueChanged(key, mergedValue); + + // Logging properties only since values could be sensitive things we don't want to log. + Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`); + + // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. + const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue, hasChanged); + + // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. + if (!hasChanged) { + return updatePromise; + } + + return Storage.mergeItem(key, batchedChanges as OnyxValue, replaceNullPatches).then(() => { + OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, validChanges, mergedValue); + return updatePromise; + }); +} + +const OnyxMerge = { + applyMerge, +}; + +export default OnyxMerge; diff --git a/lib/OnyxMerge.ts b/lib/OnyxMerge.ts new file mode 100644 index 000000000..bbf65515f --- /dev/null +++ b/lib/OnyxMerge.ts @@ -0,0 +1,35 @@ +import _ from 'underscore'; +import * as Logger from './Logger'; +import OnyxUtils from './OnyxUtils'; +import type {OnyxKey, OnyxValue} from './types'; +import cache from './OnyxCache'; +import Storage from './storage'; + +function applyMerge(key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise { + const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); + + // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. + const hasChanged = cache.hasValueChanged(key, mergedValue); + + // Logging properties only since values could be sensitive things we don't want to log. + Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`); + + // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. + const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue, hasChanged); + + // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. + if (!hasChanged) { + return updatePromise; + } + + return Storage.setItem(key, mergedValue as OnyxValue).then(() => { + OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, validChanges, mergedValue); + return updatePromise; + }); +} + +const OnyxMerge = { + applyMerge, +}; + +export default OnyxMerge; diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index e89769404..9ac4e881b 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1236,14 +1236,11 @@ function prepareKeyValuePairsForStorage( return pairs; } -function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { +function mergeChanges | undefined>(changes: TValue[], existingValue?: TValue): FastMergeResult { return applyMerge('merge', changes, existingValue); } -function mergeAndMarkChanges | undefined, TChange extends OnyxInput | undefined>( - changes: TChange[], - existingValue?: TValue, -): FastMergeResult { +function mergeAndMarkChanges | undefined>(changes: TValue[], existingValue?: TValue): FastMergeResult { return applyMerge('mark', changes, existingValue); } @@ -1253,11 +1250,7 @@ function mergeAndMarkChanges | undefined, TCha * @param changes Array of changes that should be merged * @param existingValue The existing value that should be merged with the changes */ -function applyMerge | undefined, TChange extends OnyxInput | undefined>( - mode: 'merge' | 'mark', - changes: TChange[], - existingValue?: TValue, -): FastMergeResult { +function applyMerge | undefined>(mode: 'merge' | 'mark', changes: TValue[], existingValue?: TValue): FastMergeResult { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { @@ -1266,7 +1259,7 @@ function applyMerge | undefined, TChange exten if (changes.some((change) => change && typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce>( + return changes.reduce>( (modifiedData, change) => { const options: FastMergeOptions = mode === 'merge' ? {shouldRemoveNestedNulls: true, objectRemovalMode: 'replace'} : {objectRemovalMode: 'mark'}; const {result, replaceNullPatches} = utils.fastMerge(modifiedData.result, change, options); @@ -1279,7 +1272,7 @@ function applyMerge | undefined, TChange exten return modifiedData; }, { - result: (existingValue ?? {}) as TChange, + result: (existingValue ?? {}) as TValue, replaceNullPatches: [], }, ); @@ -1287,7 +1280,7 @@ function applyMerge | undefined, TChange exten // If we have anything else we can't merge it so we'll // simply return the last value that was queued - return {result: lastChange as TChange, replaceNullPatches: []}; + return {result: lastChange as TValue, replaceNullPatches: []}; } /** From ed66dad9a1a3ecba57216c456e98784c9a942bf2 Mon Sep 17 00:00:00 2001 From: Christoph Pader Date: Wed, 21 May 2025 17:35:31 +0200 Subject: [PATCH 36/58] fix: invalid param for `sendActionToDevTools` --- lib/Onyx.ts | 7 +++-- .../index.native.ts} | 27 ++++++++++--------- lib/{OnyxMerge.ts => OnyxMerge/index.ts} | 27 ++++++++++--------- lib/OnyxMerge/types.ts | 10 +++++++ 4 files changed, 43 insertions(+), 28 deletions(-) rename lib/{OnyxMerge.native.ts => OnyxMerge/index.native.ts} (73%) rename lib/{OnyxMerge.ts => OnyxMerge/index.ts} (66%) create mode 100644 lib/OnyxMerge/types.ts diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 5bee462a1..4e4e3c4dd 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -34,7 +34,7 @@ import type {Connection} from './OnyxConnectionManager'; import connectionManager from './OnyxConnectionManager'; import * as GlobalSettings from './GlobalSettings'; import decorateWithMetrics from './metrics'; -import OnyxMerge from './OnyxMerge.native'; +import OnyxMerge from './OnyxMerge/index.native'; /** Initialize the store with actions and listening for storage events */ function init({ @@ -325,7 +325,10 @@ function merge(key: TKey, changes: OnyxMergeInput): return Promise.resolve(); } - return OnyxMerge.applyMerge(key, existingValue, validChanges); + return OnyxMerge.applyMerge(key, existingValue, validChanges).then(({mergedValue, updatePromise}) => { + OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, mergedValue); + return updatePromise; + }); } catch (error) { Logger.logAlert(`An error occurred while applying merge for key: ${key}, Error: ${error}`); return Promise.resolve(); diff --git a/lib/OnyxMerge.native.ts b/lib/OnyxMerge/index.native.ts similarity index 73% rename from lib/OnyxMerge.native.ts rename to lib/OnyxMerge/index.native.ts index f18589e77..bf323b53d 100644 --- a/lib/OnyxMerge.native.ts +++ b/lib/OnyxMerge/index.native.ts @@ -1,11 +1,12 @@ import _ from 'underscore'; -import * as Logger from './Logger'; -import OnyxUtils from './OnyxUtils'; -import type {OnyxKey, OnyxValue} from './types'; -import cache from './OnyxCache'; -import Storage from './storage'; - -function applyMerge(key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise { +import * as Logger from '../Logger'; +import OnyxUtils from '../OnyxUtils'; +import type {OnyxKey, OnyxValue} from '../types'; +import cache from '../OnyxCache'; +import Storage from '../storage'; +import type {ApplyMerge, ApplyMergeResult} from './types'; + +const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise => { // If any of the changes is null, we need to discard the existing value. const baseValue = validChanges.includes(null) ? undefined : existingValue; @@ -27,14 +28,14 @@ function applyMerge(key: TKey, existingValue: OnyxValue, replaceNullPatches).then(() => { - OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, validChanges, mergedValue); - return updatePromise; - }); -} + return Storage.mergeItem(key, batchedChanges as OnyxValue, replaceNullPatches).then(() => ({ + mergedValue, + updatePromise, + })); +}; const OnyxMerge = { applyMerge, diff --git a/lib/OnyxMerge.ts b/lib/OnyxMerge/index.ts similarity index 66% rename from lib/OnyxMerge.ts rename to lib/OnyxMerge/index.ts index bbf65515f..ea53203dd 100644 --- a/lib/OnyxMerge.ts +++ b/lib/OnyxMerge/index.ts @@ -1,11 +1,12 @@ import _ from 'underscore'; -import * as Logger from './Logger'; -import OnyxUtils from './OnyxUtils'; -import type {OnyxKey, OnyxValue} from './types'; -import cache from './OnyxCache'; -import Storage from './storage'; - -function applyMerge(key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise { +import * as Logger from '../Logger'; +import OnyxUtils from '../OnyxUtils'; +import type {OnyxKey, OnyxValue} from '../types'; +import cache from '../OnyxCache'; +import Storage from '../storage'; +import type {ApplyMerge, ApplyMergeResult} from './types'; + +const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise => { const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. @@ -19,14 +20,14 @@ function applyMerge(key: TKey, existingValue: OnyxValue).then(() => { - OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, validChanges, mergedValue); - return updatePromise; - }); -} + return Storage.setItem(key, mergedValue as OnyxValue).then(() => ({ + mergedValue, + updatePromise, + })); +}; const OnyxMerge = { applyMerge, diff --git a/lib/OnyxMerge/types.ts b/lib/OnyxMerge/types.ts new file mode 100644 index 000000000..51405c9ac --- /dev/null +++ b/lib/OnyxMerge/types.ts @@ -0,0 +1,10 @@ +import type {OnyxKey, OnyxValue} from '../types'; + +type ApplyMergeResult = { + mergedValue: OnyxValue; + updatePromise: Promise; +}; + +type ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]) => Promise; + +export type {ApplyMerge, ApplyMergeResult}; From 7d60502d36fa189138939505ccfeaeb7715b8d38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 5 Jun 2025 10:39:29 +0100 Subject: [PATCH 37/58] Fix TS and tests --- lib/OnyxUtils.ts | 19 +++++++++++++------ lib/storage/index.ts | 4 ++-- tests/perf-test/OnyxUtils.perf-test.ts | 13 +------------ tests/unit/onyxUtilsTest.ts | 2 +- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 08290ded3..3b50f1740 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1173,11 +1173,14 @@ function prepareKeyValuePairsForStorage( return pairs; } -function mergeChanges | undefined>(changes: TValue[], existingValue?: TValue): FastMergeResult { +function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { return applyMerge('merge', changes, existingValue); } -function mergeAndMarkChanges | undefined>(changes: TValue[], existingValue?: TValue): FastMergeResult { +function mergeAndMarkChanges | undefined, TChange extends OnyxInput | undefined>( + changes: TChange[], + existingValue?: TValue, +): FastMergeResult { return applyMerge('mark', changes, existingValue); } @@ -1187,7 +1190,11 @@ function mergeAndMarkChanges | undefined>(chan * @param changes Array of changes that should be merged * @param existingValue The existing value that should be merged with the changes */ -function applyMerge | undefined>(mode: 'merge' | 'mark', changes: TValue[], existingValue?: TValue): FastMergeResult { +function applyMerge | undefined, TChange extends OnyxInput | undefined>( + mode: 'merge' | 'mark', + changes: TChange[], + existingValue?: TValue, +): FastMergeResult { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { @@ -1196,7 +1203,7 @@ function applyMerge | undefined>(mode: 'merge' if (changes.some((change) => change && typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce>( + return changes.reduce>( (modifiedData, change) => { const options: FastMergeOptions = mode === 'merge' ? {shouldRemoveNestedNulls: true, objectRemovalMode: 'replace'} : {objectRemovalMode: 'mark'}; const {result, replaceNullPatches} = utils.fastMerge(modifiedData.result, change, options); @@ -1209,7 +1216,7 @@ function applyMerge | undefined>(mode: 'merge' return modifiedData; }, { - result: (existingValue ?? {}) as TValue, + result: (existingValue ?? {}) as TChange, replaceNullPatches: [], }, ); @@ -1217,7 +1224,7 @@ function applyMerge | undefined>(mode: 'merge' // If we have anything else we can't merge it so we'll // simply return the last value that was queued - return {result: lastChange as TValue, replaceNullPatches: []}; + return {result: lastChange as TChange, replaceNullPatches: []}; } /** diff --git a/lib/storage/index.ts b/lib/storage/index.ts index f9b9e2d57..41da24ce8 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -131,9 +131,9 @@ const storage: Storage = { * Multiple merging of existing and new values in a batch * This function also removes all nested null values from an object. */ - multiMerge: (pairs, mergeReplaceNullPatches) => + multiMerge: (pairs) => tryOrDegradePerformance(() => { - const promise = provider.multiMerge(pairs, mergeReplaceNullPatches); + const promise = provider.multiMerge(pairs); if (shouldKeepInstancesSync) { return promise.then(() => InstanceSync.multiMerge(pairs.map((pair) => pair[0]))); diff --git a/tests/perf-test/OnyxUtils.perf-test.ts b/tests/perf-test/OnyxUtils.perf-test.ts index 997bf8f0e..737e61bd6 100644 --- a/tests/perf-test/OnyxUtils.perf-test.ts +++ b/tests/perf-test/OnyxUtils.perf-test.ts @@ -610,15 +610,6 @@ describe('OnyxUtils', () => { }); }); - describe('removeNullValues', () => { - test('one call with one heavy object', async () => { - const key = `${collectionKey}0`; - const reportAction = mockedReportActionsMap[`${collectionKey}0`]; - - await measureFunction(() => OnyxUtils.removeNullValues(key, reportAction, true)); - }); - }); - describe('prepareKeyValuePairsForStorage', () => { test('one call with 10k heavy objects', async () => { await measureFunction(() => OnyxUtils.prepareKeyValuePairsForStorage(mockedReportActionsMap, false)); @@ -634,9 +625,7 @@ describe('OnyxUtils', () => { const changedReportAction4 = createRandomReportAction(Number(reportAction.reportActionID)); const changedReportAction5 = createRandomReportAction(Number(reportAction.reportActionID)); - await measureFunction(() => - OnyxUtils.applyMerge(reportAction, [changedReportAction1, changedReportAction2, changedReportAction3, changedReportAction4, changedReportAction5], false), - ); + await measureFunction(() => OnyxUtils.mergeChanges([changedReportAction1, changedReportAction2, changedReportAction3, changedReportAction4, changedReportAction5], reportAction)); }); }); diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index 2e065fe71..1fddf2672 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -153,7 +153,7 @@ describe('OnyxUtils', () => { describe('mergeChanges', () => { it("should return the last change if it's an array", () => { - const {result} = OnyxUtils.mergeAndMarkChanges([...testMergeChanges, [0, 1, 2]], testObject); + const {result} = OnyxUtils.mergeChanges([...testMergeChanges, [0, 1, 2]], testObject); expect(result).toEqual([0, 1, 2]); }); From d14543094b276d8a6590c2662d371baddd103d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 5 Jun 2025 11:41:18 +0100 Subject: [PATCH 38/58] More cleanup and fixes --- lib/OnyxMerge/index.native.ts | 4 ++-- lib/OnyxMerge/index.ts | 4 ++-- lib/utils.ts | 13 +++---------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/OnyxMerge/index.native.ts b/lib/OnyxMerge/index.native.ts index bf323b53d..afe7661c9 100644 --- a/lib/OnyxMerge/index.native.ts +++ b/lib/OnyxMerge/index.native.ts @@ -4,9 +4,9 @@ import OnyxUtils from '../OnyxUtils'; import type {OnyxKey, OnyxValue} from '../types'; import cache from '../OnyxCache'; import Storage from '../storage'; -import type {ApplyMerge, ApplyMergeResult} from './types'; +import type {ApplyMerge} from './types'; -const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise => { +const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]) => { // If any of the changes is null, we need to discard the existing value. const baseValue = validChanges.includes(null) ? undefined : existingValue; diff --git a/lib/OnyxMerge/index.ts b/lib/OnyxMerge/index.ts index ea53203dd..b08378884 100644 --- a/lib/OnyxMerge/index.ts +++ b/lib/OnyxMerge/index.ts @@ -4,9 +4,9 @@ import OnyxUtils from '../OnyxUtils'; import type {OnyxKey, OnyxValue} from '../types'; import cache from '../OnyxCache'; import Storage from '../storage'; -import type {ApplyMerge, ApplyMergeResult} from './types'; +import type {ApplyMerge} from './types'; -const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]): Promise => { +const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]) => { const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. diff --git a/lib/utils.ts b/lib/utils.ts index d05b0f6aa..49e044860 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -11,13 +11,10 @@ type FastMergeOptions = { /** * If set to "mark", we will mark objects that are set to null instead of simply removing them, - * so that we can batch changes together, without loosing information about the object removal. + * so that we can batch changes together, without losing information about the object removal. * If set to "replace", we will completely replace the marked objects with the new value instead of merging them. * */ objectRemovalMode?: 'mark' | 'replace' | 'none'; - - /** If true, any nested objects that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag will be completely replaced instead of merged. */ - shouldReplaceMarkedObjects?: boolean; }; type FastMergeMetadata = { @@ -128,7 +125,7 @@ function mergeObject>( // If "shouldMarkRemovedObjects" is enabled and the previous merge change (targetProperty) is null, // it means we want to fully replace this object when merging the batched changes with the Onyx value. // To achieve this, we first mark these nested objects with an internal flag. - // When calling fastMerge again with "shouldReplaceMarkedObjects" enabled, the marked objects will be removed. + // When calling fastMerge again with "mark" removal mode, the marked objects will be removed. if (options.objectRemovalMode === 'mark' && targetProperty === null) { targetProperty = {[ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true}; metadata.replaceNullPatches.push([[...basePath, key], {...sourceProperty}]); @@ -138,11 +135,8 @@ function mergeObject>( // has the internal flag set, we replace the entire destination object with the source one and remove // the flag. if (options.objectRemovalMode === 'replace' && sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { - // We do a spread here in order to have a new object reference and allow us to delete the internal flag - // of the merged object only. // eslint-disable-next-line @typescript-eslint/no-unused-vars delete sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]; - // const {ONYX_INTERNALS__REPLACE_OBJECT_MARK: _mark, ...sourcePropertyWithoutMark} = sourceProperty; destination[key] = sourceProperty; return; } @@ -158,10 +152,9 @@ function isEmptyObject(obj: T | EmptyValue): obj is EmptyValue { return typeof obj === 'object' && Object.keys(obj || {}).length === 0; } -// Mostly copied from https://medium.com/@lubaka.a/how-to-remove-lodash-performance-improvement-b306669ad0e1 - /** * Checks whether the given value can be merged. It has to be an object, but not an array, RegExp or Date. + * Mostly copied from https://medium.com/@lubaka.a/how-to-remove-lodash-performance-improvement-b306669ad0e1. */ function isMergeableObject>(value: unknown): value is TObject { const isNonNullObject = value != null ? typeof value === 'object' : false; From 7b8801fbdafe57f27eec1c6c8cd06fb6243ba79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 5 Jun 2025 15:05:13 +0100 Subject: [PATCH 39/58] More fixes --- lib/Onyx.ts | 2 +- lib/storage/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index f04bbd8c2..e8612f37c 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -33,7 +33,7 @@ import type {Connection} from './OnyxConnectionManager'; import connectionManager from './OnyxConnectionManager'; import * as GlobalSettings from './GlobalSettings'; import decorateWithMetrics from './metrics'; -import OnyxMerge from './OnyxMerge/index.native'; +import OnyxMerge from './OnyxMerge'; /** Initialize the store with actions and listening for storage events */ function init({ diff --git a/lib/storage/index.ts b/lib/storage/index.ts index 41da24ce8..97ec7ceed 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -116,9 +116,9 @@ const storage: Storage = { /** * Merging an existing value with a new one */ - mergeItem: (key, change) => + mergeItem: (key, change, replaceNullPatches) => tryOrDegradePerformance(() => { - const promise = provider.mergeItem(key, change); + const promise = provider.mergeItem(key, change, replaceNullPatches); if (shouldKeepInstancesSync) { return promise.then(() => InstanceSync.mergeItem(key)); From e577505083ecb4980eb762e31082622702c25ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 5 Jun 2025 15:08:07 +0100 Subject: [PATCH 40/58] Remove wrong assignment --- lib/OnyxUtils.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 3b50f1740..ce145ed8c 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1517,8 +1517,6 @@ GlobalSettings.addGlobalSettingsChangeListener(({enablePerformanceMetrics}) => { // @ts-expect-error Reassign getCollectionKeys = decorateWithMetrics(getCollectionKeys, 'OnyxUtils.getCollectionKeys'); // @ts-expect-error Reassign - addEvictableKeysToRecentlyAccessedList = decorateWithMetrics(cache.addEvictableKeysToRecentlyAccessedList, 'OnyxCache.addEvictableKeysToRecentlyAccessedList'); - // @ts-expect-error Reassign keysChanged = decorateWithMetrics(keysChanged, 'OnyxUtils.keysChanged'); // @ts-expect-error Reassign keyChanged = decorateWithMetrics(keyChanged, 'OnyxUtils.keyChanged'); From 6d7b634858b86d9914a99636ea604ce967094f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 5 Jun 2025 15:15:37 +0100 Subject: [PATCH 41/58] Increase Jest timeout for Reassure --- jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index 733d27f7d..4728813ba 100644 --- a/jest.config.js +++ b/jest.config.js @@ -13,5 +13,5 @@ module.exports = { timers: 'fake', testEnvironment: 'jsdom', setupFilesAfterEnv: ['@testing-library/jest-native/extend-expect', './jestSetup.js'], - testTimeout: 10000, + testTimeout: 60000, }; From 1abaaa58da664a2aae6ac55a305406ee5fd8a436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Tue, 10 Jun 2025 18:11:28 +0100 Subject: [PATCH 42/58] Tests with new removeNestedNullValues implementation --- lib/utils.ts | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/utils.ts b/lib/utils.ts index 49e044860..bee2eb73d 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -163,14 +163,34 @@ function isMergeableObject>(value: unkno /** Deep removes the nested null values from the given value. */ function removeNestedNullValues | null>(value: TValue): TValue { - if (typeof value === 'object' && !Array.isArray(value)) { - return fastMerge(value, value, { - shouldRemoveNestedNulls: true, - objectRemovalMode: 'replace', - }).result; + if (value === null || value === undefined) { + return value; } - return value; + if (typeof value !== 'object' || Array.isArray(value)) { + return value; + } + + const result: Record = {}; + + // eslint-disable-next-line no-restricted-syntax, guard-for-in + for (const key in value) { + const newValue = value[key]; + + if (newValue === null || newValue === undefined) { + // eslint-disable-next-line no-continue + continue; + } + + if (typeof newValue === 'object' && !Array.isArray(newValue)) { + const valueWithoutNestedNulls = removeNestedNullValues(newValue); + result[key] = valueWithoutNestedNulls; + } else { + result[key] = newValue; + } + } + + return result as TValue; } /** Formats the action name by uppercasing and adding the key if provided. */ From 46d1230ad669ba8d481f6c90f868803091142ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Tue, 10 Jun 2025 18:22:15 +0100 Subject: [PATCH 43/58] Re-run Reassure --- lib/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils.ts b/lib/utils.ts index bee2eb73d..a618ce0a1 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -161,7 +161,7 @@ function isMergeableObject>(value: unkno return isNonNullObject && !(value instanceof RegExp) && !(value instanceof Date) && !Array.isArray(value); } -/** Deep removes the nested null values from the given value. */ +/** Deep removes the nested null values from the given value. */ function removeNestedNullValues | null>(value: TValue): TValue { if (value === null || value === undefined) { return value; From 7fca303bd52a6a0840da522b5a1fa1be9d0c15ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Tue, 10 Jun 2025 19:02:42 +0100 Subject: [PATCH 44/58] Re-run Reassure --- lib/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils.ts b/lib/utils.ts index a618ce0a1..bee2eb73d 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -161,7 +161,7 @@ function isMergeableObject>(value: unkno return isNonNullObject && !(value instanceof RegExp) && !(value instanceof Date) && !Array.isArray(value); } -/** Deep removes the nested null values from the given value. */ +/** Deep removes the nested null values from the given value. */ function removeNestedNullValues | null>(value: TValue): TValue { if (value === null || value === undefined) { return value; From ab9be045282576fe50a0e6d178135d27ab63d696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Wed, 18 Jun 2025 18:19:43 +0100 Subject: [PATCH 45/58] Minor fix to mergeObject and add test for mergeCollection --- lib/storage/providers/IDBKeyValProvider.ts | 1 + lib/storage/providers/MemoryOnlyProvider.ts | 3 +- lib/utils.ts | 8 ++- tests/unit/onyxTest.ts | 80 +++++++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/lib/storage/providers/IDBKeyValProvider.ts b/lib/storage/providers/IDBKeyValProvider.ts index 64d419a83..0fd2a63ec 100644 --- a/lib/storage/providers/IDBKeyValProvider.ts +++ b/lib/storage/providers/IDBKeyValProvider.ts @@ -53,6 +53,7 @@ const provider: StorageProvider = { shouldRemoveNestedNulls: true, objectRemovalMode: 'replace', }).result; + return promisifyRequest(store.put(newValue, key)); }); return Promise.all(upsertMany).then(() => undefined); diff --git a/lib/storage/providers/MemoryOnlyProvider.ts b/lib/storage/providers/MemoryOnlyProvider.ts index 1510d20cb..2367e972e 100644 --- a/lib/storage/providers/MemoryOnlyProvider.ts +++ b/lib/storage/providers/MemoryOnlyProvider.ts @@ -86,10 +86,11 @@ const provider: StorageProvider = { multiMerge(pairs) { _.forEach(pairs, ([key, value]) => { const existingValue = store[key] as Record; + const newValue = utils.fastMerge(existingValue, value as Record, { shouldRemoveNestedNulls: true, objectRemovalMode: 'replace', - }).result as OnyxValue; + }).result; set(key, newValue); }); diff --git a/lib/utils.ts b/lib/utils.ts index bee2eb73d..17df6c108 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -135,9 +135,11 @@ function mergeObject>( // has the internal flag set, we replace the entire destination object with the source one and remove // the flag. if (options.objectRemovalMode === 'replace' && sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - delete sourceProperty[ONYX_INTERNALS__REPLACE_OBJECT_MARK]; - destination[key] = sourceProperty; + // We do a spread here in order to have a new object reference and allow us to delete the internal flag + // of the merged object only. + const sourcePropertyWithoutMark = {...sourceProperty}; + delete sourcePropertyWithoutMark.ONYX_INTERNALS__REPLACE_OBJECT_MARK; + destination[key] = sourcePropertyWithoutMark; return; } diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index 1644e2079..f21760391 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -1930,6 +1930,86 @@ describe('Onyx', () => { expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); }); + + describe('mergeCollection', () => { + it('replacing old object after null merge', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord = { + sub_entry1: { + id: 'sub_entry1', + someKey: 'someValue', + }, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry2: DeepRecord = { + sub_entry2: { + id: 'sub_entry2', + someKey: 'someValue', + }, + }; + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1}); + await Onyx.multiSet({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: entry2}); + + const entry1ExpectedResult = lodashCloneDeep(entry1); + const entry2ExpectedResult = lodashCloneDeep(entry2); + const queuedUpdates: OnyxUpdate[] = []; + + queuedUpdates.push( + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + value: { + // Removing the "sub_entry1" object in this update. + // Any subsequent changes to this object should completely replace the existing object in store. + sub_entry1: null, + }, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + onyxMethod: 'merge', + value: { + // Removing the "sub_entry2" object in this update. + // Any subsequent changes to this object should completely replace the existing object in store. + sub_entry2: null, + }, + }, + ); + delete entry1ExpectedResult.sub_entry1; + delete entry2ExpectedResult.sub_entry2; + + queuedUpdates.push( + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + value: { + // This change should completely replace "sub_entry1" existing object in store. + sub_entry1: { + newKey: 'newValue', + }, + }, + }, + { + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, + onyxMethod: 'merge', + value: { + // This change should completely replace "sub_entry2" existing object in store. + sub_entry2: { + newKey: 'newValue', + }, + }, + }, + ); + entry1ExpectedResult.sub_entry1 = {newKey: 'newValue'}; + entry2ExpectedResult.sub_entry2 = {newKey: 'newValue'}; + + await Onyx.update(queuedUpdates); + + expect(result).toEqual({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult, + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: entry2ExpectedResult, + }); + }); + }); }); describe('merge', () => { From d731a72a9918be6dce8efa2e7dffc580998c3041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Wed, 18 Jun 2025 20:23:09 +0100 Subject: [PATCH 46/58] Add StorageMock checks for each test and make Jest always use OnyxMerge web version --- jest.config.js | 4 ++++ tests/unit/onyxTest.ts | 19 ++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/jest.config.js b/jest.config.js index 4fc0584dd..790e5f050 100644 --- a/jest.config.js +++ b/jest.config.js @@ -14,4 +14,8 @@ module.exports = { setupFilesAfterEnv: ['./jestSetup.js'], testTimeout: 60000, transformIgnorePatterns: ['node_modules/(?!((@)?react-native|@ngneat/falso|uuid)/)'], + moduleNameMapper: { + // Redirect all imports of OnyxMerge to its web version during unit tests. + '^(.*)/OnyxMerge$': '/lib/OnyxMerge/index.ts', + }, }; diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index f21760391..4f74a2e35 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -4,6 +4,7 @@ import Onyx from '../../lib'; import waitForPromisesToResolve from '../utils/waitForPromisesToResolve'; import OnyxUtils from '../../lib/OnyxUtils'; import type OnyxCache from '../../lib/OnyxCache'; +import StorageMock from '../../lib/storage'; import type {DeepRecord, OnyxCollection, OnyxUpdate} from '../../lib/types'; import type GenericCollection from '../utils/GenericCollection'; import type {Connection} from '../../lib/OnyxConnectionManager'; @@ -1738,11 +1739,8 @@ describe('Onyx', () => { await Onyx.update(queuedUpdates); - expect(result).toEqual({ - [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { - someKey: 'someValueChanged', - }, - }); + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: {someKey: 'someValueChanged'}}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual({someKey: 'someValueChanged'}); }); describe('should replace the old value after a null merge in a nested property when batching updates', () => { @@ -1797,6 +1795,7 @@ describe('Onyx', () => { await Onyx.update(queuedUpdates); expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); }); it('setting new object after null merge', async () => { @@ -1869,6 +1868,7 @@ describe('Onyx', () => { await Onyx.update(queuedUpdates); expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); }); it('setting new object after null merge of a primitive property', async () => { @@ -1929,6 +1929,7 @@ describe('Onyx', () => { await Onyx.update(queuedUpdates); expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); }); describe('mergeCollection', () => { @@ -2008,6 +2009,10 @@ describe('Onyx', () => { [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult, [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`]: entry2ExpectedResult, }); + expect(await StorageMock.multiGet([`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`])).toEqual([ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, entry1ExpectedResult], + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry2`, entry2ExpectedResult], + ]); }); }); }); @@ -2042,6 +2047,7 @@ describe('Onyx', () => { await waitForPromisesToResolve(); expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: {someKey: 'someValueChanged'}}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual({someKey: 'someValueChanged'}); }); describe('should replace the old value after a null merge in a nested property when batching merges', () => { @@ -2087,6 +2093,7 @@ describe('Onyx', () => { await waitForPromisesToResolve(); expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); }); it('setting new object after null merge', async () => { @@ -2146,6 +2153,7 @@ describe('Onyx', () => { await waitForPromisesToResolve(); expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); }); it('setting new object after null merge of a primitive property', async () => { @@ -2197,6 +2205,7 @@ describe('Onyx', () => { await waitForPromisesToResolve(); expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); }); }); From 7f3c4d9a5eb0317e9f04e8c852654656c5b40db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 23 Jun 2025 19:18:58 +0100 Subject: [PATCH 47/58] Fix Onyx.update() to use mergeChanges instead of marking --- lib/Onyx.ts | 2 +- tests/unit/onyxTest.ts | 68 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index e8612f37c..1a179b3ca 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -716,7 +716,7 @@ function update(data: OnyxUpdate[]): Promise { Object.entries(updateQueue).forEach(([key, operations]) => { if (operations[0] === null) { - const batchedChanges = OnyxUtils.mergeAndMarkChanges(operations).result; + const batchedChanges = OnyxUtils.mergeChanges(operations).result; promises.push(() => set(key, batchedChanges)); return; } diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index 4f74a2e35..7927ef212 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -1932,6 +1932,74 @@ describe('Onyx', () => { expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); }); + it('replacing nested object during updates', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const entry1: DeepRecord | undefined = { + id: 'entry1', + someKey: 'someValue', + }; + await Onyx.multiSet({ + [`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: { + id: 'entry1', + someKey: 'someValue', + }, + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let entry1ExpectedResult = lodashCloneDeep(entry1) as DeepRecord | undefined; + const queuedUpdates: OnyxUpdate[] = []; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + // Removing the entire object in this update. + // Any subsequent changes to this key should completely replace the old value. + value: null, + }); + entry1ExpectedResult = undefined; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + // This change should completely replace `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1` old value. + value: { + someKey: 'someValueChanged', + someNestedObject: { + someNestedKey: 'someNestedValue', + }, + }, + }); + entry1ExpectedResult = {someKey: 'someValueChanged', someNestedObject: {someNestedKey: 'someNestedValue'}}; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + value: { + // Removing the "sub_entry1" object in this update. + // Any subsequent changes to this key should completely replace the old update's value. + someNestedObject: null, + }, + }); + delete entry1ExpectedResult.someNestedObject; + + queuedUpdates.push({ + key: `${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`, + onyxMethod: 'merge', + // This change should completely replace `someNestedObject` old update's value. + value: { + someNestedObject: { + someNestedKeyChanged: 'someNestedValueChange', + }, + }, + }); + entry1ExpectedResult.someNestedObject = {someNestedKeyChanged: 'someNestedValueChange'}; + + await Onyx.update(queuedUpdates); + + expect(result).toEqual({[`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`]: entry1ExpectedResult}); + expect(await StorageMock.getItem(`${ONYX_KEYS.COLLECTION.TEST_UPDATE}entry1`)).toEqual(entry1ExpectedResult); + }); + describe('mergeCollection', () => { it('replacing old object after null merge', async () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any From 8af4d447b3e17f585650965c951b1b185d89e275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 23 Jun 2025 20:41:06 +0100 Subject: [PATCH 48/58] Add more comments --- API-INTERNAL.md | 101 +++++++++--------------- API.md | 5 +- lib/Onyx.ts | 2 + lib/OnyxMerge/index.native.ts | 2 + lib/OnyxMerge/index.ts | 1 + lib/OnyxUtils.ts | 19 ++++- lib/storage/providers/SQLiteProvider.ts | 20 +++-- lib/types.ts | 8 +- lib/utils.ts | 9 ++- tests/unit/fastMergeTest.ts | 2 +- 10 files changed, 92 insertions(+), 77 deletions(-) diff --git a/API-INTERNAL.md b/API-INTERNAL.md index d413c0219..3f615487e 100644 --- a/API-INTERNAL.md +++ b/API-INTERNAL.md @@ -17,9 +17,6 @@

getDeferredInitTask()

Getter - returns the deffered init task.

-
getEvictionBlocklist()
-

Getter - returns the eviction block list.

-
getSkippableCollectionMemberIDs()

Getter - returns the skippable collection member IDs.

@@ -71,9 +68,6 @@ is associated with a collection of keys.

Checks to see if a provided key is the exact configured key of our connected subscriber or if the provided key is a collection member key (in case our configured key is a "collection key")

-
isEvictableKey()
-

Checks to see if this key has been flagged as safe for removal.

-
getCollectionKey(key) ⇒

Extracts the collection identifier of a given collection member key.

For example:

@@ -88,20 +82,6 @@ or if the provided key is a collection member key (in case our configured key is

Tries to get a value from the cache. If the value is not present in cache it will return the default value or undefined. If the requested key is a collection, it will return an object with all the collection members.

-
removeLastAccessedKey()
-

Remove a key from the recently accessed key list.

-
-
addLastAccessedKey()
-

Add a key to the list of recently accessed keys. The least -recently accessed key should be at the head and the most -recently accessed key at the tail.

-
-
addEvictableKeysToRecentlyAccessedList()
-

Take all the keys that are safe to evict and add them to -the recently accessed list when initializing the app. This -enables keys that have not recently been accessed to be -removed.

-
keysChanged()

When a collection of keys change, search for any callbacks matching the collection key and trigger those callbacks

@@ -144,8 +124,15 @@ whatever it is we attempted to do.

This method transforms an object like {'@MyApp_user': myUserValue, '@MyApp_key': myKeyValue} to an array of key-value pairs in the above format and removes key-value pairs that are being set to null

-
mergeChanges(changes)
-

Merges an array of changes with an existing value or creates a single change

+
mergeChanges(changes, existingValue)
+

Merges an array of changes with an existing value or creates a single change.

+
+
mergeAndMarkChanges(changes, existingValue)
+

Merges an array of changes with an existing value or creates a single change. +It will also mark deep nested objects that need to be entirely replaced during the merge.

+
+
applyMerge(changes, existingValue)
+

Merges an array of changes with an existing value or creates a single change.

initializeWithDefaultKeyStates()

Merge user provided default key value pairs.

@@ -187,12 +174,6 @@ Getter - returns the default key states. ## getDeferredInitTask() Getter - returns the deffered init task. -**Kind**: global function - - -## getEvictionBlocklist() -Getter - returns the eviction block list. - **Kind**: global function @@ -217,7 +198,7 @@ Sets the initial values for the Onyx store | --- | --- | | keys | `ONYXKEYS` constants object from Onyx.init() | | initialKeyStates | initial data to set when `init()` and `clear()` are called | -| evictableKeys | This is an array of keys (individual or collection patterns) that are eligible for automatic removal when storage limits are reached. | +| evictableKeys | This is an array of keys (individual or collection patterns) that when provided to Onyx are flagged as "safe" for removal. | @@ -313,12 +294,6 @@ or throws an Error if the key is not a collection one. Checks to see if a provided key is the exact configured key of our connected subscriber or if the provided key is a collection member key (in case our configured key is a "collection key") -**Kind**: global function - - -## isEvictableKey() -Checks to see if this key has been flagged as safe for removal. - **Kind**: global function @@ -344,29 +319,6 @@ For example: Tries to get a value from the cache. If the value is not present in cache it will return the default value or undefined. If the requested key is a collection, it will return an object with all the collection members. -**Kind**: global function - - -## removeLastAccessedKey() -Remove a key from the recently accessed key list. - -**Kind**: global function - - -## addLastAccessedKey() -Add a key to the list of recently accessed keys. The least -recently accessed key should be at the head and the most -recently accessed key at the tail. - -**Kind**: global function - - -## addEvictableKeysToRecentlyAccessedList() -Take all the keys that are safe to evict and add them to -the recently accessed list when initializing the app. This -enables keys that have not recently been accessed to be -removed. - **Kind**: global function @@ -460,7 +412,6 @@ whatever it is we attempted to do. Notifies subscribers and writes current value to cache **Kind**: global function -**Returns**: The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely ## prepareKeyValuePairsForStorage() ⇒ @@ -472,14 +423,40 @@ to an array of key-value pairs in the above format and removes key-value pairs t **Returns**: an array of key - value pairs <[key, value]> -## mergeChanges(changes) -Merges an array of changes with an existing value or creates a single change +## mergeChanges(changes, existingValue) +Merges an array of changes with an existing value or creates a single change. + +**Kind**: global function + +| Param | Description | +| --- | --- | +| changes | Array of changes that should be merged | +| existingValue | The existing value that should be merged with the changes | + + + +## mergeAndMarkChanges(changes, existingValue) +Merges an array of changes with an existing value or creates a single change. +It will also mark deep nested objects that need to be entirely replaced during the merge. + +**Kind**: global function + +| Param | Description | +| --- | --- | +| changes | Array of changes that should be merged | +| existingValue | The existing value that should be merged with the changes | + + + +## applyMerge(changes, existingValue) +Merges an array of changes with an existing value or creates a single change. **Kind**: global function | Param | Description | | --- | --- | -| changes | Array of changes that should be applied to the existing value | +| changes | Array of changes that should be merged | +| existingValue | The existing value that should be merged with the changes | diff --git a/API.md b/API.md index dc1fa112f..ca3a38ce6 100644 --- a/API.md +++ b/API.md @@ -28,7 +28,7 @@ Values of type Object get merged with the old value, whilst for Onyx.set() calls do not work this way so use caution when mixing Onyx.merge() and Onyx.set().

-
mergeCollection(collectionKey, collection)
+
mergeCollection(collectionKey, collection, mergeReplaceNullPatches)

Merges a collection based on their keys

clear(keysToPreserve)
@@ -157,7 +157,7 @@ Onyx.merge(ONYXKEYS.POLICY, {name: 'My Workspace'}); // -> {id: 1, name: 'My Wor ``` -## mergeCollection(collectionKey, collection) +## mergeCollection(collectionKey, collection, mergeReplaceNullPatches) Merges a collection based on their keys **Kind**: global function @@ -166,6 +166,7 @@ Merges a collection based on their keys | --- | --- | | collectionKey | e.g. `ONYXKEYS.COLLECTION.REPORT` | | collection | Object collection keyed by individual collection member keys and values | +| mergeReplaceNullPatches | Record where the key is a collection member key and the value is a list of tuples that we'll use to replace the nested objects of that collection member record with something else. | **Example** ```js diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 1a179b3ca..04d6505f3 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -356,6 +356,8 @@ function merge(key: TKey, changes: OnyxMergeInput): * * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` * @param collection Object collection keyed by individual collection member keys and values + * @param mergeReplaceNullPatches Record where the key is a collection member key and the value is a list of + * tuples that we'll use to replace the nested objects of that collection member record with something else. */ function mergeCollection( collectionKey: TKey, diff --git a/lib/OnyxMerge/index.native.ts b/lib/OnyxMerge/index.native.ts index afe7661c9..c56db56f9 100644 --- a/lib/OnyxMerge/index.native.ts +++ b/lib/OnyxMerge/index.native.ts @@ -31,6 +31,8 @@ const applyMerge: ApplyMerge = (key: TKey, existingValue: return Promise.resolve({mergedValue, updatePromise}); } + // For native platforms we use `mergeItem` that will take advantage of JSON_PATCH and JSON_REPLACE SQL operations to + // merge the object in a performant way. return Storage.mergeItem(key, batchedChanges as OnyxValue, replaceNullPatches).then(() => ({ mergedValue, updatePromise, diff --git a/lib/OnyxMerge/index.ts b/lib/OnyxMerge/index.ts index b08378884..c5b5a3676 100644 --- a/lib/OnyxMerge/index.ts +++ b/lib/OnyxMerge/index.ts @@ -23,6 +23,7 @@ const applyMerge: ApplyMerge = (key: TKey, existingValue: return Promise.resolve({mergedValue, updatePromise}); } + // For web platforms we use `setItem` since the object was already merged with its changes before. return Storage.setItem(key, mergedValue as OnyxValue).then(() => ({ mergedValue, updatePromise, diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index ce145ed8c..b9df9c897 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1147,8 +1147,8 @@ function hasPendingMergeForKey(key: OnyxKey): boolean { * Storage expects array like: [["@MyApp_user", value_1], ["@MyApp_key", value_2]] * This method transforms an object like {'@MyApp_user': myUserValue, '@MyApp_key': myKeyValue} * to an array of key-value pairs in the above format and removes key-value pairs that are being set to null - -* @return an array of key - value pairs <[key, value]> + * + * @return an array of key - value pairs <[key, value]> */ function prepareKeyValuePairsForStorage( data: Record>, @@ -1173,10 +1173,23 @@ function prepareKeyValuePairsForStorage( return pairs; } +/** + * Merges an array of changes with an existing value or creates a single change. + * + * @param changes Array of changes that should be merged + * @param existingValue The existing value that should be merged with the changes + */ function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { return applyMerge('merge', changes, existingValue); } +/** + * Merges an array of changes with an existing value or creates a single change. + * It will also mark deep nested objects that need to be entirely replaced during the merge. + * + * @param changes Array of changes that should be merged + * @param existingValue The existing value that should be merged with the changes + */ function mergeAndMarkChanges | undefined, TChange extends OnyxInput | undefined>( changes: TChange[], existingValue?: TValue, @@ -1185,7 +1198,7 @@ function mergeAndMarkChanges | undefined, TCha } /** - * Merges an array of changes with an existing value or creates a single change + * Merges an array of changes with an existing value or creates a single change. * * @param changes Array of changes that should be merged * @param existingValue The existing value that should be merged with the changes diff --git a/lib/storage/providers/SQLiteProvider.ts b/lib/storage/providers/SQLiteProvider.ts index 2183d8d04..859176505 100644 --- a/lib/storage/providers/SQLiteProvider.ts +++ b/lib/storage/providers/SQLiteProvider.ts @@ -44,11 +44,17 @@ type PageCountResult = { const DB_NAME = 'OnyxDB'; let db: NitroSQLiteConnection; -function replacer(key: string, value: unknown) { +/** + * Prevents the stringifying of the object markers. + */ +function objectMarkRemover(key: string, value: unknown) { if (key === utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK) return undefined; return value; } +/** + * Transforms the replace null patches into SQL queries to be passed to JSON_REPLACE. + */ function generateJSONReplaceSQLQueries(key: string, patches: FastMergeReplaceNullPatch[]): string[][] { const queries = patches.map(([pathArray, value]) => { const jsonPath = `$.${pathArray.join('.')}`; @@ -114,12 +120,15 @@ const provider: StorageProvider = { multiMerge(pairs) { const commands: BatchQueryCommand[] = []; + // Query to merge the change into the DB value. const patchQuery = `INSERT INTO keyvaluepairs (record_key, valueJSON) VALUES (:key, JSON(:value)) ON CONFLICT DO UPDATE SET valueJSON = JSON_PATCH(valueJSON, JSON(:value)); `; const patchQueryArguments: string[][] = []; + + // Query to fully replace the nested objects of the DB value. const replaceQuery = `UPDATE keyvaluepairs SET valueJSON = JSON_REPLACE(valueJSON, ?, JSON(?)) WHERE record_key = ?; @@ -128,12 +137,9 @@ const provider: StorageProvider = { const nonNullishPairs = pairs.filter((pair) => pair[1] !== undefined); - // eslint-disable-next-line @typescript-eslint/prefer-for-of - for (let i = 0; i < nonNullishPairs.length; i++) { - const [key, value, replaceNullPatches] = nonNullishPairs[i]; - - const valueAfterReplace = JSON.stringify(value, replacer); - patchQueryArguments.push([key, valueAfterReplace]); + for (const [key, value, replaceNullPatches] of nonNullishPairs) { + const changeWithoutMarkers = JSON.stringify(value, objectMarkRemover); + patchQueryArguments.push([key, changeWithoutMarkers]); const patches = replaceNullPatches ?? []; if (patches.length > 0) { diff --git a/lib/types.ts b/lib/types.ts index b6cd8a1af..bb78e6ed3 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -486,7 +486,13 @@ type InitOptions = { // eslint-disable-next-line @typescript-eslint/no-explicit-any type GenericFunction = (...args: any[]) => any; -type MultiMergeReplaceNullPatches = {[TKey in OnyxKey]: FastMergeReplaceNullPatch[]}; +/** + * Represents a record where the key is a collection member key and the value is a list of + * tuples that we'll use to replace the nested objects of that collection member record with something else. + */ +type MultiMergeReplaceNullPatches = { + [TKey in OnyxKey]: FastMergeReplaceNullPatch[]; +}; /** * Represents a combination of Merge and Set operations that should be executed in Onyx diff --git a/lib/utils.ts b/lib/utils.ts index 17df6c108..616080521 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -3,6 +3,13 @@ import type {ConnectOptions, OnyxInput, OnyxKey} from './types'; type EmptyObject = Record; type EmptyValue = EmptyObject | null | undefined; +/** + * A tuple where the first value is the path to the nested object that contains the + * internal `ONYX_INTERNALS__REPLACE_OBJECT_MARK` flag, and the second value is the data we want to replace + * in that path. + * + * This tuple will be used in SQLiteProvider to replace the nested object using `JSON_REPLACE`. + * */ type FastMergeReplaceNullPatch = [string[], unknown]; type FastMergeOptions = { @@ -18,7 +25,7 @@ type FastMergeOptions = { }; type FastMergeMetadata = { - /** The path to the object that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag. */ + /** The list of tuples that will be used in SQLiteProvider to replace the nested objects using `JSON_REPLACE`. */ replaceNullPatches: FastMergeReplaceNullPatch[]; }; diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index 8821bdb1c..43d91cf01 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -138,7 +138,7 @@ describe('fastMerge', () => { expect(result.replaceNullPatches).toEqual([[['b', 'd'], {h: 'h'}]]); }); - it('should completely replace the target object with its source when the source has the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag and "shouldReplaceMarkedObjects" is true', () => { + it('should completely replace the target object with its source when the source has the "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag and "objectRemovalMode" is set to "replace"', () => { const result = utils.fastMerge( testObject, { From b14f7eb1dbc17c1bdfeda58e40cc4ad7486bdb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 23 Jun 2025 20:48:50 +0100 Subject: [PATCH 49/58] Improve types --- lib/types.ts | 4 +++ lib/utils.ts | 2 +- tests/unit/fastMergeTest.ts | 4 +-- tests/unit/onyxTest.ts | 35 ++++++++---------------- tests/unit/onyxUtilsTest.ts | 8 +++--- tests/utils/collections/reportActions.ts | 5 ++-- 6 files changed, 25 insertions(+), 33 deletions(-) diff --git a/lib/types.ts b/lib/types.ts index bb78e6ed3..ba4fffa02 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -486,6 +486,9 @@ type InitOptions = { // eslint-disable-next-line @typescript-eslint/no-explicit-any type GenericFunction = (...args: any[]) => any; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type GenericDeepRecord = DeepRecord; + /** * Represents a record where the key is a collection member key and the value is a list of * tuples that we'll use to replace the nested objects of that collection member record with something else. @@ -543,4 +546,5 @@ export type { WithOnyxConnectOptions, MultiMergeReplaceNullPatches, MixedOperationsQueue, + GenericDeepRecord, }; diff --git a/lib/utils.ts b/lib/utils.ts index 616080521..824978b4e 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -20,7 +20,7 @@ type FastMergeOptions = { * If set to "mark", we will mark objects that are set to null instead of simply removing them, * so that we can batch changes together, without losing information about the object removal. * If set to "replace", we will completely replace the marked objects with the new value instead of merging them. - * */ + */ objectRemovalMode?: 'mark' | 'replace' | 'none'; }; diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index 43d91cf01..d2b137f43 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -1,7 +1,7 @@ -import type {DeepRecord} from '../../lib/types'; +import type {GenericDeepRecord} from '../../lib/types'; import utils from '../../lib/utils'; -type DeepObject = DeepRecord | unknown[]; +type DeepObject = GenericDeepRecord | unknown[]; const testObject: DeepObject = { a: 'a', diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index 7927ef212..946e5bf91 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -5,7 +5,7 @@ import waitForPromisesToResolve from '../utils/waitForPromisesToResolve'; import OnyxUtils from '../../lib/OnyxUtils'; import type OnyxCache from '../../lib/OnyxCache'; import StorageMock from '../../lib/storage'; -import type {DeepRecord, OnyxCollection, OnyxUpdate} from '../../lib/types'; +import type {GenericDeepRecord, OnyxCollection, OnyxUpdate} from '../../lib/types'; import type GenericCollection from '../utils/GenericCollection'; import type {Connection} from '../../lib/OnyxConnectionManager'; @@ -858,7 +858,6 @@ describe('Onyx', () => { // When we pass it to Onyx.update // @ts-expect-error This is an invalid call to Onyx.update Onyx.update(data); - // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error) { if (error instanceof Error) { // Then we should expect the error message below @@ -875,7 +874,6 @@ describe('Onyx', () => { // When we pass it to Onyx.update // @ts-expect-error This is an invalid call to Onyx.update Onyx.update(data); - // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error) { if (error instanceof Error) { // Then we should expect the error message below @@ -1757,8 +1755,7 @@ describe('Onyx', () => { }); it('replacing old object after null merge', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord = { + const entry1: GenericDeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', @@ -1799,8 +1796,7 @@ describe('Onyx', () => { }); it('setting new object after null merge', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord = { + const entry1: GenericDeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', @@ -1872,8 +1868,7 @@ describe('Onyx', () => { }); it('setting new object after null merge of a primitive property', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord = { + const entry1: GenericDeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', @@ -1933,8 +1928,7 @@ describe('Onyx', () => { }); it('replacing nested object during updates', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord | undefined = { + const entry1: GenericDeepRecord | undefined = { id: 'entry1', someKey: 'someValue', }; @@ -1945,8 +1939,7 @@ describe('Onyx', () => { }, }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let entry1ExpectedResult = lodashCloneDeep(entry1) as DeepRecord | undefined; + let entry1ExpectedResult = lodashCloneDeep(entry1) as GenericDeepRecord | undefined; const queuedUpdates: OnyxUpdate[] = []; queuedUpdates.push({ @@ -2002,15 +1995,14 @@ describe('Onyx', () => { describe('mergeCollection', () => { it('replacing old object after null merge', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord = { + const entry1: GenericDeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', }, }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry2: DeepRecord = { + + const entry2: GenericDeepRecord = { sub_entry2: { id: 'sub_entry2', someKey: 'someValue', @@ -2132,8 +2124,7 @@ describe('Onyx', () => { }); it('replacing old object after null merge', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord = { + const entry1: GenericDeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', @@ -2165,8 +2156,7 @@ describe('Onyx', () => { }); it('setting new object after null merge', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord = { + const entry1: GenericDeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', @@ -2225,8 +2215,7 @@ describe('Onyx', () => { }); it('setting new object after null merge of a primitive property', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const entry1: DeepRecord = { + const entry1: GenericDeepRecord = { sub_entry1: { id: 'sub_entry1', someKey: 'someValue', diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index 1fddf2672..e0f8390a3 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -1,9 +1,9 @@ import Onyx from '../../lib'; import OnyxUtils from '../../lib/OnyxUtils'; -import type {DeepRecord} from '../../lib/types'; +import type {GenericDeepRecord} from '../../lib/types'; import utils from '../../lib/utils'; -const testObject: DeepRecord = { +const testObject: GenericDeepRecord = { a: 'a', b: { c: 'c', @@ -15,7 +15,7 @@ const testObject: DeepRecord = { }, }; -const testMergeChanges: Array> = [ +const testMergeChanges: GenericDeepRecord[] = [ { b: { d: { @@ -165,7 +165,7 @@ describe('OnyxUtils', () => { }); it('should merge data correctly when applying batched changes', () => { - const batchedChanges: DeepRecord = { + const batchedChanges: GenericDeepRecord = { b: { d: { i: 'i', diff --git a/tests/utils/collections/reportActions.ts b/tests/utils/collections/reportActions.ts index 4fae80338..d139e6520 100644 --- a/tests/utils/collections/reportActions.ts +++ b/tests/utils/collections/reportActions.ts @@ -1,7 +1,7 @@ import {randAggregation, randBoolean, randWord} from '@ngneat/falso'; import {format} from 'date-fns'; import {createCollection} from './createCollection'; -import type {DeepRecord} from '../../../lib/types'; +import type {GenericDeepRecord} from '../../../lib/types'; const getRandomDate = (): string => { const randomTimestamp = Math.random() * new Date().getTime(); @@ -19,8 +19,7 @@ const getRandomReportActions = (collection: string, length = 10000) => length, ); -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export default function createRandomReportAction(index: number): DeepRecord { +export default function createRandomReportAction(index: number): GenericDeepRecord { return { actionName: randWord(), reportActionID: index.toString(), From bd8d67e31f0da848468d253b2a6a5b21efd18add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 23 Jun 2025 20:57:27 +0100 Subject: [PATCH 50/58] Minor fixes --- tests/unit/fastMergeTest.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index d2b137f43..4e3581e0e 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -103,8 +103,7 @@ describe('fastMerge', () => { }); it('should replace an object with an array', () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = utils.fastMerge(testObject, [1, 2, 3] as any, { + const result = utils.fastMerge(testObject, [1, 2, 3], { shouldRemoveNestedNulls: true, }); @@ -112,8 +111,7 @@ describe('fastMerge', () => { }); it('should replace an array with an object', () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = utils.fastMerge([1, 2, 3] as any, testObject, { + const result = utils.fastMerge([1, 2, 3], testObject, { shouldRemoveNestedNulls: true, }); From 38a5f13629b2dc3ab2bb630a84d22cf4870e0509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 23 Jun 2025 21:00:39 +0100 Subject: [PATCH 51/58] Remove useless test --- tests/unit/onyxUtilsTest.ts | 53 +------------------------------------ 1 file changed, 1 insertion(+), 52 deletions(-) diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index e0f8390a3..cd8417362 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -200,7 +200,7 @@ describe('OnyxUtils', () => { }); describe('mergeAndMarkChanges', () => { - it('should apply the replacement markers if the we have properties with objects being removed and added back during the changes', () => { + it('should apply the replacement markers if we have properties with objects being removed and added back during the changes', () => { const {result, replaceNullPatches} = OnyxUtils.mergeAndMarkChanges(testMergeChanges); expect(result).toEqual({ @@ -223,56 +223,5 @@ describe('OnyxUtils', () => { [['b', 'g'], {k: 'k'}], ]); }); - - it('should 2', () => { - const {result, replaceNullPatches} = OnyxUtils.mergeAndMarkChanges([ - { - // Removing the "originalMessage" object in this update. - // Any subsequent changes to this object should completely replace the existing object in store. - originalMessage: null, - }, - { - // This change should completely replace "originalMessage" existing object in store. - originalMessage: { - errorMessage: 'newErrorMessage', - }, - receipt: { - // Removing the "nestedObject" object in this update. - // Any subsequent changes to this object should completely replace the existing object in store. - nestedObject: null, - }, - }, - { - receipt: { - receiptID: null, - filename: 'newFilename', - // This change should completely replace "receipt" existing object in store. - nestedObject: { - nestedKey2: 'newNestedKey2', - }, - }, - }, - ]); - - expect(result).toEqual({ - originalMessage: { - errorMessage: 'newErrorMessage', - [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, - }, - receipt: { - receiptID: null, - filename: 'newFilename', - nestedObject: { - nestedKey2: 'newNestedKey2', - [utils.ONYX_INTERNALS__REPLACE_OBJECT_MARK]: true, - }, - }, - }); - - expect(replaceNullPatches).toEqual([ - [['originalMessage'], {errorMessage: 'newErrorMessage'}], - [['receipt', 'nestedObject'], {nestedKey2: 'newNestedKey2'}], - ]); - }); }); }); From a27d3c81b85336c91b8811d1a3d2986db6e9d3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Mon, 30 Jun 2025 11:13:58 +0100 Subject: [PATCH 52/58] Apply feedback --- lib/OnyxUtils.ts | 6 +++--- lib/utils.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 682363b0d..b9616a470 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -1182,7 +1182,7 @@ function prepareKeyValuePairsForStorage( * @param existingValue The existing value that should be merged with the changes */ function mergeChanges | undefined, TChange extends OnyxInput | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult { - return applyMerge('merge', changes, existingValue); + return mergeInternal('merge', changes, existingValue); } /** @@ -1196,7 +1196,7 @@ function mergeAndMarkChanges | undefined, TCha changes: TChange[], existingValue?: TValue, ): FastMergeResult { - return applyMerge('mark', changes, existingValue); + return mergeInternal('mark', changes, existingValue); } /** @@ -1205,7 +1205,7 @@ function mergeAndMarkChanges | undefined, TCha * @param changes Array of changes that should be merged * @param existingValue The existing value that should be merged with the changes */ -function applyMerge | undefined, TChange extends OnyxInput | undefined>( +function mergeInternal | undefined, TChange extends OnyxInput | undefined>( mode: 'merge' | 'mark', changes: TChange[], existingValue?: TValue, diff --git a/lib/utils.ts b/lib/utils.ts index 824978b4e..8bf36211e 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -123,7 +123,7 @@ function mergeObject>( return; } - // If source value is not a mergable object, we need to set the source value it directly. + // If the source value is not a mergable object, we need to set the key directly. if (!isMergeableObject(sourceProperty)) { destination[key] = sourceProperty; return; From 32c725d54ef185570a03a14e11ba3ad50c1ae5ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Wed, 2 Jul 2025 15:37:20 +0100 Subject: [PATCH 53/58] Address comments --- lib/utils.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/utils.ts b/lib/utils.ts index 8bf36211e..c8e69e898 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -33,7 +33,7 @@ type FastMergeResult = { /** The result of the merge. */ result: TValue; - /** The path to the object that contains the internal "ONYX_INTERNALS__REPLACE_OBJECT_MARK" flag. */ + /** The list of tuples that will be used in SQLiteProvider to replace the nested objects using `JSON_REPLACE`. */ replaceNullPatches: FastMergeReplaceNullPatch[]; }; @@ -116,7 +116,7 @@ function mergeObject>( const sourceProperty = source?.[key] as Record; // If "shouldRemoveNestedNulls" is true, we want to remove (nested) null values from the merged object. - // If either the source value is null, we want to omit the key from the merged object. + // If the source value is null, we want to omit the key from the merged object. const shouldOmitNullishProperty = options.shouldRemoveNestedNulls && sourceProperty === null; if (sourceProperty === undefined || shouldOmitNullishProperty) { @@ -184,18 +184,18 @@ function removeNestedNullValues | null>(value: // eslint-disable-next-line no-restricted-syntax, guard-for-in for (const key in value) { - const newValue = value[key]; + const propertyValue = value[key]; - if (newValue === null || newValue === undefined) { + if (propertyValue === null || propertyValue === undefined) { // eslint-disable-next-line no-continue continue; } - if (typeof newValue === 'object' && !Array.isArray(newValue)) { - const valueWithoutNestedNulls = removeNestedNullValues(newValue); + if (typeof propertyValue === 'object' && !Array.isArray(propertyValue)) { + const valueWithoutNestedNulls = removeNestedNullValues(propertyValue); result[key] = valueWithoutNestedNulls; } else { - result[key] = newValue; + result[key] = propertyValue; } } From 90d90001e6f8206179790c0198e641c08a2dd98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Wed, 2 Jul 2025 15:38:05 +0100 Subject: [PATCH 54/58] Build docs --- API-INTERNAL.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/API-INTERNAL.md b/API-INTERNAL.md index 3f615487e..5a71533de 100644 --- a/API-INTERNAL.md +++ b/API-INTERNAL.md @@ -23,7 +23,7 @@
setSkippableCollectionMemberIDs()

Setter - sets the skippable collection member IDs.

-
initStoreValues(keys, initialKeyStates, evictableKeys)
+
initStoreValues(keys, initialKeyStates, evictableKeys, fullyMergedSnapshotKeys)

Sets the initial values for the Onyx store

maybeFlushBatchUpdates()
@@ -131,7 +131,7 @@ to an array of key-value pairs in the above format and removes key-value pairs t

Merges an array of changes with an existing value or creates a single change. It will also mark deep nested objects that need to be entirely replaced during the merge.

-
applyMerge(changes, existingValue)
+
mergeInternal(changes, existingValue)

Merges an array of changes with an existing value or creates a single change.

initializeWithDefaultKeyStates()
@@ -149,6 +149,9 @@ It will also mark deep nested objects that need to be entirely replaced during t
unsubscribeFromKey(subscriptionID)

Disconnects and removes the listener from the Onyx key.

+
clearOnyxUtilsInternals()
+

Clear internal variables used in this file, useful in test environments.

+
@@ -189,7 +192,7 @@ Setter - sets the skippable collection member IDs. **Kind**: global function -## initStoreValues(keys, initialKeyStates, evictableKeys) +## initStoreValues(keys, initialKeyStates, evictableKeys, fullyMergedSnapshotKeys) Sets the initial values for the Onyx store **Kind**: global function @@ -199,6 +202,7 @@ Sets the initial values for the Onyx store | keys | `ONYXKEYS` constants object from Onyx.init() | | initialKeyStates | initial data to set when `init()` and `clear()` are called | | evictableKeys | This is an array of keys (individual or collection patterns) that when provided to Onyx are flagged as "safe" for removal. | +| fullyMergedSnapshotKeys | Array of snapshot collection keys where full merge is supported and data structure can be changed after merge. | @@ -446,9 +450,9 @@ It will also mark deep nested objects that need to be entirely replaced during t | changes | Array of changes that should be merged | | existingValue | The existing value that should be merged with the changes | - + -## applyMerge(changes, existingValue) +## mergeInternal(changes, existingValue) Merges an array of changes with an existing value or creates a single change. **Kind**: global function @@ -499,3 +503,9 @@ Disconnects and removes the listener from the Onyx key. | --- | --- | | subscriptionID | Subscription ID returned by calling `OnyxUtils.subscribeToKey()`. | + + +## clearOnyxUtilsInternals() +Clear internal variables used in this file, useful in test environments. + +**Kind**: global function From 79b018d39655655d8160f91eba0092a354e26363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Wed, 2 Jul 2025 18:10:09 +0100 Subject: [PATCH 55/58] Move GenericDeepRecord to tests directory --- lib/types.ts | 4 ---- tests/types.ts | 12 ++++++++++++ tests/unit/fastMergeTest.ts | 2 +- tests/unit/onyxTest.ts | 3 ++- tests/unit/onyxUtilsTest.ts | 2 +- tests/utils/collections/reportActions.ts | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 tests/types.ts diff --git a/lib/types.ts b/lib/types.ts index 46a5b2431..5af416974 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -494,9 +494,6 @@ type InitOptions = { // eslint-disable-next-line @typescript-eslint/no-explicit-any type GenericFunction = (...args: any[]) => any; -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type GenericDeepRecord = DeepRecord; - /** * Represents a record where the key is a collection member key and the value is a list of * tuples that we'll use to replace the nested objects of that collection member record with something else. @@ -554,5 +551,4 @@ export type { WithOnyxConnectOptions, MultiMergeReplaceNullPatches, MixedOperationsQueue, - GenericDeepRecord, }; diff --git a/tests/types.ts b/tests/types.ts new file mode 100644 index 000000000..1aef55855 --- /dev/null +++ b/tests/types.ts @@ -0,0 +1,12 @@ +import type {DeepRecord} from '../lib/types'; + +// The types declared inside this file should be used only for testing. + +/** + * Utility type to represent a object that can accept any value and contain any deep objects. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type GenericDeepRecord = DeepRecord; + +// eslint-disable-next-line import/prefer-default-export +export type {GenericDeepRecord}; diff --git a/tests/unit/fastMergeTest.ts b/tests/unit/fastMergeTest.ts index 4e3581e0e..0823db572 100644 --- a/tests/unit/fastMergeTest.ts +++ b/tests/unit/fastMergeTest.ts @@ -1,5 +1,5 @@ -import type {GenericDeepRecord} from '../../lib/types'; import utils from '../../lib/utils'; +import type {GenericDeepRecord} from '../types'; type DeepObject = GenericDeepRecord | unknown[]; diff --git a/tests/unit/onyxTest.ts b/tests/unit/onyxTest.ts index b5c22e0d3..5f4604950 100644 --- a/tests/unit/onyxTest.ts +++ b/tests/unit/onyxTest.ts @@ -5,7 +5,8 @@ import waitForPromisesToResolve from '../utils/waitForPromisesToResolve'; import OnyxUtils from '../../lib/OnyxUtils'; import type OnyxCache from '../../lib/OnyxCache'; import StorageMock from '../../lib/storage'; -import type {GenericDeepRecord, OnyxCollection, OnyxUpdate} from '../../lib/types'; +import type {OnyxCollection, OnyxUpdate} from '../../lib/types'; +import type {GenericDeepRecord} from '../types'; import type GenericCollection from '../utils/GenericCollection'; import type {Connection} from '../../lib/OnyxConnectionManager'; diff --git a/tests/unit/onyxUtilsTest.ts b/tests/unit/onyxUtilsTest.ts index cd8417362..521e28454 100644 --- a/tests/unit/onyxUtilsTest.ts +++ b/tests/unit/onyxUtilsTest.ts @@ -1,6 +1,6 @@ import Onyx from '../../lib'; import OnyxUtils from '../../lib/OnyxUtils'; -import type {GenericDeepRecord} from '../../lib/types'; +import type {GenericDeepRecord} from '../types'; import utils from '../../lib/utils'; const testObject: GenericDeepRecord = { diff --git a/tests/utils/collections/reportActions.ts b/tests/utils/collections/reportActions.ts index d139e6520..240fa654a 100644 --- a/tests/utils/collections/reportActions.ts +++ b/tests/utils/collections/reportActions.ts @@ -1,7 +1,7 @@ import {randAggregation, randBoolean, randWord} from '@ngneat/falso'; import {format} from 'date-fns'; import {createCollection} from './createCollection'; -import type {GenericDeepRecord} from '../../../lib/types'; +import type {GenericDeepRecord} from '../../types'; const getRandomDate = (): string => { const randomTimestamp = Math.random() * new Date().getTime(); From c6719747adc8dda2c14253b9c8bcebab20d45dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Wed, 2 Jul 2025 18:13:09 +0100 Subject: [PATCH 56/58] Rollback removeNestedNullValues refactor --- lib/utils.ts | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/lib/utils.ts b/lib/utils.ts index c8e69e898..5fc15e755 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -172,34 +172,14 @@ function isMergeableObject>(value: unkno /** Deep removes the nested null values from the given value. */ function removeNestedNullValues | null>(value: TValue): TValue { - if (value === null || value === undefined) { - return value; + if (typeof value === 'object' && !Array.isArray(value)) { + return fastMerge(value, value, { + shouldRemoveNestedNulls: true, + objectRemovalMode: 'replace', + }).result; } - if (typeof value !== 'object' || Array.isArray(value)) { - return value; - } - - const result: Record = {}; - - // eslint-disable-next-line no-restricted-syntax, guard-for-in - for (const key in value) { - const propertyValue = value[key]; - - if (propertyValue === null || propertyValue === undefined) { - // eslint-disable-next-line no-continue - continue; - } - - if (typeof propertyValue === 'object' && !Array.isArray(propertyValue)) { - const valueWithoutNestedNulls = removeNestedNullValues(propertyValue); - result[key] = valueWithoutNestedNulls; - } else { - result[key] = propertyValue; - } - } - - return result as TValue; + return value; } /** Formats the action name by uppercasing and adding the key if provided. */ From 466feb125eb63d0ec4ca56f44618b502d49b9f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 3 Jul 2025 17:07:33 +0100 Subject: [PATCH 57/58] Address comments --- lib/Onyx.ts | 7 +++---- lib/OnyxMerge/index.native.ts | 14 ++++++++------ lib/OnyxMerge/index.ts | 14 ++++++++------ lib/OnyxMerge/types.ts | 12 ++++++++---- lib/OnyxUtils.ts | 11 +++++++++++ tests/perf-test/OnyxUtils.perf-test.ts | 2 +- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 2cf4becb5..75f474549 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1,4 +1,3 @@ -import _ from 'underscore'; import * as Logger from './Logger'; import cache, {TASK} from './OnyxCache'; import * as PerformanceUtils from './PerformanceUtils'; @@ -176,14 +175,14 @@ function set(key: TKey, value: OnyxSetInput): Promis // Therefore, we don't need to further broadcast and update the value so we can return early. if (value === null) { OnyxUtils.remove(key); - Logger.logInfo(`set called for key: ${key} => null passed, so key was removed`); + OnyxUtils.logKeyRemoved(OnyxUtils.METHOD.SET, key); return Promise.resolve(); } const valueWithoutNestedNullValues = utils.removeNestedNullValues(value) as OnyxValue; const hasChanged = cache.hasValueChanged(key, valueWithoutNestedNullValues); - Logger.logInfo(`set called for key: ${key}${_.isObject(value) ? ` properties: ${_.keys(value).join(',')}` : ''} hasChanged: ${hasChanged}`); + OnyxUtils.logKeyChanged(OnyxUtils.METHOD.SET, key, value, hasChanged); // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. const updatePromise = OnyxUtils.broadcastUpdate(key, valueWithoutNestedNullValues, hasChanged); @@ -327,8 +326,8 @@ function merge(key: TKey, changes: OnyxMergeInput): // If the last change is null, we can just delete the key. // Therefore, we don't need to further broadcast and update the value so we can return early. if (validChanges.at(-1) === null) { - Logger.logInfo(`merge called for key: ${key} => null passed, so key was removed`); OnyxUtils.remove(key); + OnyxUtils.logKeyRemoved(OnyxUtils.METHOD.MERGE, key); return Promise.resolve(); } diff --git a/lib/OnyxMerge/index.native.ts b/lib/OnyxMerge/index.native.ts index c56db56f9..b796dfde4 100644 --- a/lib/OnyxMerge/index.native.ts +++ b/lib/OnyxMerge/index.native.ts @@ -1,14 +1,16 @@ -import _ from 'underscore'; -import * as Logger from '../Logger'; import OnyxUtils from '../OnyxUtils'; -import type {OnyxKey, OnyxValue} from '../types'; +import type {OnyxInput, OnyxKey, OnyxValue} from '../types'; import cache from '../OnyxCache'; import Storage from '../storage'; import type {ApplyMerge} from './types'; -const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]) => { +const applyMerge: ApplyMerge = | undefined, TChange extends OnyxInput | undefined>( + key: TKey, + existingValue: TValue, + validChanges: TChange[], +) => { // If any of the changes is null, we need to discard the existing value. - const baseValue = validChanges.includes(null) ? undefined : existingValue; + const baseValue = validChanges.includes(null as TChange) ? undefined : existingValue; // We first batch the changes into a single change with object removal marks, // so that SQLite can merge the changes more efficiently. @@ -21,7 +23,7 @@ const applyMerge: ApplyMerge = (key: TKey, existingValue: const hasChanged = cache.hasValueChanged(key, mergedValue); // Logging properties only since values could be sensitive things we don't want to log. - Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`); + OnyxUtils.logKeyChanged(OnyxUtils.METHOD.MERGE, key, mergedValue, hasChanged); // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue, hasChanged); diff --git a/lib/OnyxMerge/index.ts b/lib/OnyxMerge/index.ts index c5b5a3676..2a648499b 100644 --- a/lib/OnyxMerge/index.ts +++ b/lib/OnyxMerge/index.ts @@ -1,19 +1,21 @@ -import _ from 'underscore'; -import * as Logger from '../Logger'; -import OnyxUtils from '../OnyxUtils'; -import type {OnyxKey, OnyxValue} from '../types'; import cache from '../OnyxCache'; +import OnyxUtils from '../OnyxUtils'; import Storage from '../storage'; +import type {OnyxInput, OnyxKey, OnyxValue} from '../types'; import type {ApplyMerge} from './types'; -const applyMerge: ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]) => { +const applyMerge: ApplyMerge = | undefined, TChange extends OnyxInput | undefined>( + key: TKey, + existingValue: TValue, + validChanges: TChange[], +) => { const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue); // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge. const hasChanged = cache.hasValueChanged(key, mergedValue); // Logging properties only since values could be sensitive things we don't want to log. - Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`); + OnyxUtils.logKeyChanged(OnyxUtils.METHOD.MERGE, key, mergedValue, hasChanged); // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue, hasChanged); diff --git a/lib/OnyxMerge/types.ts b/lib/OnyxMerge/types.ts index 51405c9ac..c59b7892a 100644 --- a/lib/OnyxMerge/types.ts +++ b/lib/OnyxMerge/types.ts @@ -1,10 +1,14 @@ -import type {OnyxKey, OnyxValue} from '../types'; +import type {OnyxInput, OnyxKey} from '../types'; -type ApplyMergeResult = { - mergedValue: OnyxValue; +type ApplyMergeResult = { + mergedValue: TValue; updatePromise: Promise; }; -type ApplyMerge = (key: TKey, existingValue: OnyxValue, validChanges: unknown[]) => Promise; +type ApplyMerge = | undefined, TChange extends OnyxInput | null>( + key: TKey, + existingValue: TValue, + validChanges: TChange[], +) => Promise>; export type {ApplyMerge, ApplyMergeResult}; diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index c092a201d..29ecc08a4 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -3,6 +3,7 @@ import {deepEqual} from 'fast-equals'; import lodashClone from 'lodash/clone'; import type {ValueOf} from 'type-fest'; import lodashPick from 'lodash/pick'; +import _ from 'underscore'; import DevTools from './DevTools'; import * as Logger from './Logger'; import type Onyx from './Onyx'; @@ -1508,6 +1509,14 @@ function updateSnapshots(data: OnyxUpdate[], mergeFn: typeof Onyx.merge): Array< return promises; } +function logKeyChanged(onyxMethod: Extract, key: OnyxKey, value: unknown, hasChanged: boolean) { + Logger.logInfo(`${onyxMethod} called for key: ${key}${_.isObject(value) ? ` properties: ${_.keys(value).join(',')}` : ''} hasChanged: ${hasChanged}`); +} + +function logKeyRemoved(onyxMethod: Extract, key: OnyxKey) { + Logger.logInfo(`${onyxMethod} called for key: ${key} => null passed, so key was removed`); +} + /** * Clear internal variables used in this file, useful in test environments. */ @@ -1569,6 +1578,8 @@ const OnyxUtils = { addKeyToRecentlyAccessedIfNeeded, reduceCollectionWithSelector, updateSnapshots, + logKeyChanged, + logKeyRemoved, }; GlobalSettings.addGlobalSettingsChangeListener(({enablePerformanceMetrics}) => { diff --git a/tests/perf-test/OnyxUtils.perf-test.ts b/tests/perf-test/OnyxUtils.perf-test.ts index e17f86981..9b6f0bcaf 100644 --- a/tests/perf-test/OnyxUtils.perf-test.ts +++ b/tests/perf-test/OnyxUtils.perf-test.ts @@ -616,7 +616,7 @@ describe('OnyxUtils', () => { }); }); - describe('applyMerge', () => { + describe('mergeChanges', () => { test('one call merging 5 changes', async () => { const reportAction = mockedReportActionsMap[`${collectionKey}0`]; const changedReportAction1 = createRandomReportAction(Number(reportAction.reportActionID)); From 405f255be11d12a3ed0872d5a7449fb31ac2da98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Henriques?= Date: Thu, 3 Jul 2025 17:47:08 +0100 Subject: [PATCH 58/58] Implement internal mergeCollectionWithPatches function --- API-INTERNAL.md | 20 +++++++ API.md | 9 ++- lib/Onyx.ts | 129 +++---------------------------------------- lib/OnyxUtils.ts | 140 +++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 167 insertions(+), 131 deletions(-) diff --git a/API-INTERNAL.md b/API-INTERNAL.md index 5a71533de..8da520402 100644 --- a/API-INTERNAL.md +++ b/API-INTERNAL.md @@ -149,6 +149,11 @@ It will also mark deep nested objects that need to be entirely replaced during t
unsubscribeFromKey(subscriptionID)

Disconnects and removes the listener from the Onyx key.

+
mergeCollectionWithPatches(collectionKey, collection, mergeReplaceNullPatches)
+

Merges a collection based on their keys. +Serves as core implementation for Onyx.mergeCollection() public function, the difference being +that this internal function allows passing an additional mergeReplaceNullPatches parameter.

+
clearOnyxUtilsInternals()

Clear internal variables used in this file, useful in test environments.

@@ -503,6 +508,21 @@ Disconnects and removes the listener from the Onyx key. | --- | --- | | subscriptionID | Subscription ID returned by calling `OnyxUtils.subscribeToKey()`. | + + +## mergeCollectionWithPatches(collectionKey, collection, mergeReplaceNullPatches) +Merges a collection based on their keys. +Serves as core implementation for `Onyx.mergeCollection()` public function, the difference being +that this internal function allows passing an additional `mergeReplaceNullPatches` parameter. + +**Kind**: global function + +| Param | Description | +| --- | --- | +| collectionKey | e.g. `ONYXKEYS.COLLECTION.REPORT` | +| collection | Object collection keyed by individual collection member keys and values | +| mergeReplaceNullPatches | Record where the key is a collection member key and the value is a list of tuples that we'll use to replace the nested objects of that collection member record with something else. | + ## clearOnyxUtilsInternals() diff --git a/API.md b/API.md index ca3a38ce6..ec083bbbf 100644 --- a/API.md +++ b/API.md @@ -28,8 +28,8 @@ Values of type Object get merged with the old value, whilst for Onyx.set() calls do not work this way so use caution when mixing Onyx.merge() and Onyx.set().

-
mergeCollection(collectionKey, collection, mergeReplaceNullPatches)
-

Merges a collection based on their keys

+
mergeCollection(collectionKey, collection)
+

Merges a collection based on their keys.

clear(keysToPreserve)

Clear out all the data in the store

@@ -157,8 +157,8 @@ Onyx.merge(ONYXKEYS.POLICY, {name: 'My Workspace'}); // -> {id: 1, name: 'My Wor ``` -## mergeCollection(collectionKey, collection, mergeReplaceNullPatches) -Merges a collection based on their keys +## mergeCollection(collectionKey, collection) +Merges a collection based on their keys. **Kind**: global function @@ -166,7 +166,6 @@ Merges a collection based on their keys | --- | --- | | collectionKey | e.g. `ONYXKEYS.COLLECTION.REPORT` | | collection | Object collection keyed by individual collection member keys and values | -| mergeReplaceNullPatches | Record where the key is a collection member key and the value is a list of tuples that we'll use to replace the nested objects of that collection member record with something else. | **Example** ```js diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 75f474549..a656f4fad 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -24,7 +24,6 @@ import type { OnyxValue, OnyxInput, OnyxMethodMap, - MultiMergeReplaceNullPatches, } from './types'; import OnyxUtils from './OnyxUtils'; import logMessages from './logMessages'; @@ -345,7 +344,7 @@ function merge(key: TKey, changes: OnyxMergeInput): } /** - * Merges a collection based on their keys + * Merges a collection based on their keys. * * @example * @@ -356,125 +355,9 @@ function merge(key: TKey, changes: OnyxMergeInput): * * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` * @param collection Object collection keyed by individual collection member keys and values - * @param mergeReplaceNullPatches Record where the key is a collection member key and the value is a list of - * tuples that we'll use to replace the nested objects of that collection member record with something else. */ -function mergeCollection( - collectionKey: TKey, - collection: OnyxMergeCollectionInput, - mergeReplaceNullPatches?: MultiMergeReplaceNullPatches, -): Promise { - if (!OnyxUtils.isValidNonEmptyCollectionForMerge(collection)) { - Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.'); - return Promise.resolve(); - } - - let resultCollection: OnyxInputKeyValueMapping = collection; - let resultCollectionKeys = Object.keys(resultCollection); - - // Confirm all the collection keys belong to the same parent - if (!OnyxUtils.doAllCollectionItemsBelongToSameParent(collectionKey, resultCollectionKeys)) { - return Promise.resolve(); - } - - const skippableCollectionMemberIDs = OnyxUtils.getSkippableCollectionMemberIDs(); - if (skippableCollectionMemberIDs.size) { - resultCollection = resultCollectionKeys.reduce((result: OnyxInputKeyValueMapping, key) => { - try { - const [, collectionMemberID] = OnyxUtils.splitCollectionMemberKey(key, collectionKey); - // If the collection member key is a skippable one we set its value to null. - // eslint-disable-next-line no-param-reassign - result[key] = !skippableCollectionMemberIDs.has(collectionMemberID) ? resultCollection[key] : null; - } catch { - // Something went wrong during split, so we assign the data to result anyway. - // eslint-disable-next-line no-param-reassign - result[key] = resultCollection[key]; - } - - return result; - }, {}); - } - resultCollectionKeys = Object.keys(resultCollection); - - return OnyxUtils.getAllKeys() - .then((persistedKeys) => { - // Split to keys that exist in storage and keys that don't - const keys = resultCollectionKeys.filter((key) => { - if (resultCollection[key] === null) { - OnyxUtils.remove(key); - return false; - } - return true; - }); - - const existingKeys = keys.filter((key) => persistedKeys.has(key)); - - const cachedCollectionForExistingKeys = OnyxUtils.getCachedCollection(collectionKey, existingKeys); - - const existingKeyCollection = existingKeys.reduce((obj: OnyxInputKeyValueMapping, key) => { - const {isCompatible, existingValueType, newValueType} = utils.checkCompatibilityWithExistingValue(resultCollection[key], cachedCollectionForExistingKeys[key]); - - if (!isCompatible) { - Logger.logAlert(logMessages.incompatibleUpdateAlert(key, 'mergeCollection', existingValueType, newValueType)); - return obj; - } - - // eslint-disable-next-line no-param-reassign - obj[key] = resultCollection[key]; - return obj; - }, {}) as Record>; - - const newCollection: Record> = {}; - keys.forEach((key) => { - if (persistedKeys.has(key)) { - return; - } - newCollection[key] = resultCollection[key]; - }); - - // When (multi-)merging the values with the existing values in storage, - // we don't want to remove nested null values from the data that we pass to the storage layer, - // because the storage layer uses them to remove nested keys from storage natively. - const keyValuePairsForExistingCollection = OnyxUtils.prepareKeyValuePairsForStorage(existingKeyCollection, false, mergeReplaceNullPatches); - - // We can safely remove nested null values when using (multi-)set, - // because we will simply overwrite the existing values in storage. - const keyValuePairsForNewCollection = OnyxUtils.prepareKeyValuePairsForStorage(newCollection, true); - - const promises = []; - - // We need to get the previously existing values so we can compare the new ones - // against them, to avoid unnecessary subscriber updates. - const previousCollectionPromise = Promise.all(existingKeys.map((key) => OnyxUtils.get(key).then((value) => [key, value]))).then(Object.fromEntries); - - // New keys will be added via multiSet while existing keys will be updated using multiMerge - // This is because setting a key that doesn't exist yet with multiMerge will throw errors - if (keyValuePairsForExistingCollection.length > 0) { - promises.push(Storage.multiMerge(keyValuePairsForExistingCollection)); - } - - if (keyValuePairsForNewCollection.length > 0) { - promises.push(Storage.multiSet(keyValuePairsForNewCollection)); - } - - // finalMergedCollection contains all the keys that were merged, without the keys of incompatible updates - const finalMergedCollection = {...existingKeyCollection, ...newCollection}; - - // Prefill cache if necessary by calling get() on any existing keys and then merge original data to cache - // and update all subscribers - const promiseUpdate = previousCollectionPromise.then((previousCollection) => { - cache.merge(finalMergedCollection); - return OnyxUtils.scheduleNotifyCollectionSubscribers(collectionKey, finalMergedCollection, previousCollection); - }); - - return Promise.all(promises) - .catch((error) => OnyxUtils.evictStorageAndRetry(error, mergeCollection, collectionKey, resultCollection)) - .then(() => { - OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE_COLLECTION, undefined, resultCollection); - return promiseUpdate; - }); - }) - .then(() => undefined); +function mergeCollection(collectionKey: TKey, collection: OnyxMergeCollectionInput): Promise { + return OnyxUtils.mergeCollectionWithPatches(collectionKey, collection); } /** @@ -708,7 +591,11 @@ function update(data: OnyxUpdate[]): Promise { if (!utils.isEmptyObject(batchedCollectionUpdates.merge)) { promises.push(() => - mergeCollection(collectionKey, batchedCollectionUpdates.merge as Collection, batchedCollectionUpdates.mergeReplaceNullPatches), + OnyxUtils.mergeCollectionWithPatches( + collectionKey, + batchedCollectionUpdates.merge as Collection, + batchedCollectionUpdates.mergeReplaceNullPatches, + ), ); } if (!utils.isEmptyObject(batchedCollectionUpdates.set)) { diff --git a/lib/OnyxUtils.ts b/lib/OnyxUtils.ts index 29ecc08a4..6e7581577 100644 --- a/lib/OnyxUtils.ts +++ b/lib/OnyxUtils.ts @@ -25,6 +25,7 @@ import type { OnyxCollection, OnyxEntry, OnyxInput, + OnyxInputKeyValueMapping, OnyxKey, OnyxMergeCollectionInput, OnyxUpdate, @@ -39,6 +40,7 @@ import createDeferredTask from './createDeferredTask'; import * as GlobalSettings from './GlobalSettings'; import decorateWithMetrics from './metrics'; import type {StorageKeyValuePair} from './storage/providers/types'; +import logMessages from './logMessages'; // Method constants const METHOD = { @@ -361,7 +363,7 @@ function multiGet(keys: CollectionKeyBase[]): Promise { if (skippableCollectionMemberIDs.size) { try { - const [, collectionMemberID] = OnyxUtils.splitCollectionMemberKey(key); + const [, collectionMemberID] = splitCollectionMemberKey(key); if (skippableCollectionMemberIDs.has(collectionMemberID)) { // The key is a skippable one, so we skip this iteration. return; @@ -387,7 +389,7 @@ function multiGet(keys: CollectionKeyBase[]): Promise|OnyxEntry>`, which is not what we want. This preserves the order of the keys provided. */ function tupleGet(keys: Keys): Promise<{[Index in keyof Keys]: OnyxValue}> { - return Promise.all(keys.map((key) => OnyxUtils.get(key))) as Promise<{[Index in keyof Keys]: OnyxValue}>; + return Promise.all(keys.map((key) => get(key))) as Promise<{[Index in keyof Keys]: OnyxValue}>; } /** @@ -1444,12 +1446,12 @@ function unsubscribeFromKey(subscriptionID: number): void { } function updateSnapshots(data: OnyxUpdate[], mergeFn: typeof Onyx.merge): Array<() => Promise> { - const snapshotCollectionKey = OnyxUtils.getSnapshotKey(); + const snapshotCollectionKey = getSnapshotKey(); if (!snapshotCollectionKey) return []; const promises: Array<() => Promise> = []; - const snapshotCollection = OnyxUtils.getCachedCollection(snapshotCollectionKey); + const snapshotCollection = getCachedCollection(snapshotCollectionKey); Object.entries(snapshotCollection).forEach(([snapshotEntryKey, snapshotEntryValue]) => { // Snapshots may not be present in cache. We don't know how to update them so we skip. @@ -1461,7 +1463,7 @@ function updateSnapshots(data: OnyxUpdate[], mergeFn: typeof Onyx.merge): Array< data.forEach(({key, value}) => { // snapshots are normal keys so we want to skip update if they are written to Onyx - if (OnyxUtils.isCollectionMemberKey(snapshotCollectionKey, key)) { + if (isCollectionMemberKey(snapshotCollectionKey, key)) { return; } @@ -1509,6 +1511,133 @@ function updateSnapshots(data: OnyxUpdate[], mergeFn: typeof Onyx.merge): Array< return promises; } +/** + * Merges a collection based on their keys. + * Serves as core implementation for `Onyx.mergeCollection()` public function, the difference being + * that this internal function allows passing an additional `mergeReplaceNullPatches` parameter. + * + * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` + * @param collection Object collection keyed by individual collection member keys and values + * @param mergeReplaceNullPatches Record where the key is a collection member key and the value is a list of + * tuples that we'll use to replace the nested objects of that collection member record with something else. + */ +function mergeCollectionWithPatches( + collectionKey: TKey, + collection: OnyxMergeCollectionInput, + mergeReplaceNullPatches?: MultiMergeReplaceNullPatches, +): Promise { + if (!isValidNonEmptyCollectionForMerge(collection)) { + Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.'); + return Promise.resolve(); + } + + let resultCollection: OnyxInputKeyValueMapping = collection; + let resultCollectionKeys = Object.keys(resultCollection); + + // Confirm all the collection keys belong to the same parent + if (!doAllCollectionItemsBelongToSameParent(collectionKey, resultCollectionKeys)) { + return Promise.resolve(); + } + + if (skippableCollectionMemberIDs.size) { + resultCollection = resultCollectionKeys.reduce((result: OnyxInputKeyValueMapping, key) => { + try { + const [, collectionMemberID] = splitCollectionMemberKey(key, collectionKey); + // If the collection member key is a skippable one we set its value to null. + // eslint-disable-next-line no-param-reassign + result[key] = !skippableCollectionMemberIDs.has(collectionMemberID) ? resultCollection[key] : null; + } catch { + // Something went wrong during split, so we assign the data to result anyway. + // eslint-disable-next-line no-param-reassign + result[key] = resultCollection[key]; + } + + return result; + }, {}); + } + resultCollectionKeys = Object.keys(resultCollection); + + return getAllKeys() + .then((persistedKeys) => { + // Split to keys that exist in storage and keys that don't + const keys = resultCollectionKeys.filter((key) => { + if (resultCollection[key] === null) { + remove(key); + return false; + } + return true; + }); + + const existingKeys = keys.filter((key) => persistedKeys.has(key)); + + const cachedCollectionForExistingKeys = getCachedCollection(collectionKey, existingKeys); + + const existingKeyCollection = existingKeys.reduce((obj: OnyxInputKeyValueMapping, key) => { + const {isCompatible, existingValueType, newValueType} = utils.checkCompatibilityWithExistingValue(resultCollection[key], cachedCollectionForExistingKeys[key]); + + if (!isCompatible) { + Logger.logAlert(logMessages.incompatibleUpdateAlert(key, 'mergeCollection', existingValueType, newValueType)); + return obj; + } + + // eslint-disable-next-line no-param-reassign + obj[key] = resultCollection[key]; + return obj; + }, {}) as Record>; + + const newCollection: Record> = {}; + keys.forEach((key) => { + if (persistedKeys.has(key)) { + return; + } + newCollection[key] = resultCollection[key]; + }); + + // When (multi-)merging the values with the existing values in storage, + // we don't want to remove nested null values from the data that we pass to the storage layer, + // because the storage layer uses them to remove nested keys from storage natively. + const keyValuePairsForExistingCollection = prepareKeyValuePairsForStorage(existingKeyCollection, false, mergeReplaceNullPatches); + + // We can safely remove nested null values when using (multi-)set, + // because we will simply overwrite the existing values in storage. + const keyValuePairsForNewCollection = prepareKeyValuePairsForStorage(newCollection, true); + + const promises = []; + + // We need to get the previously existing values so we can compare the new ones + // against them, to avoid unnecessary subscriber updates. + const previousCollectionPromise = Promise.all(existingKeys.map((key) => get(key).then((value) => [key, value]))).then(Object.fromEntries); + + // New keys will be added via multiSet while existing keys will be updated using multiMerge + // This is because setting a key that doesn't exist yet with multiMerge will throw errors + if (keyValuePairsForExistingCollection.length > 0) { + promises.push(Storage.multiMerge(keyValuePairsForExistingCollection)); + } + + if (keyValuePairsForNewCollection.length > 0) { + promises.push(Storage.multiSet(keyValuePairsForNewCollection)); + } + + // finalMergedCollection contains all the keys that were merged, without the keys of incompatible updates + const finalMergedCollection = {...existingKeyCollection, ...newCollection}; + + // Prefill cache if necessary by calling get() on any existing keys and then merge original data to cache + // and update all subscribers + const promiseUpdate = previousCollectionPromise.then((previousCollection) => { + cache.merge(finalMergedCollection); + return scheduleNotifyCollectionSubscribers(collectionKey, finalMergedCollection, previousCollection); + }); + + return Promise.all(promises) + .catch((error) => evictStorageAndRetry(error, mergeCollectionWithPatches, collectionKey, resultCollection)) + .then(() => { + sendActionToDevTools(METHOD.MERGE_COLLECTION, undefined, resultCollection); + return promiseUpdate; + }); + }) + .then(() => undefined); +} + function logKeyChanged(onyxMethod: Extract, key: OnyxKey, value: unknown, hasChanged: boolean) { Logger.logInfo(`${onyxMethod} called for key: ${key}${_.isObject(value) ? ` properties: ${_.keys(value).join(',')}` : ''} hasChanged: ${hasChanged}`); } @@ -1578,6 +1707,7 @@ const OnyxUtils = { addKeyToRecentlyAccessedIfNeeded, reduceCollectionWithSelector, updateSnapshots, + mergeCollectionWithPatches, logKeyChanged, logKeyRemoved, };