From 6892ae3a6141d88b7ea04cb37119b3f27320fdfa Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Sat, 1 Nov 2025 13:31:33 +0100 Subject: [PATCH 01/23] rm comment --- inc/ti/qbind.t.h | 1 - 1 file changed, 1 deletion(-) diff --git a/inc/ti/qbind.t.h b/inc/ti/qbind.t.h index b81ac4f4..f3ef4392 100644 --- a/inc/ti/qbind.t.h +++ b/inc/ti/qbind.t.h @@ -23,7 +23,6 @@ typedef enum typedef enum { - /* first three flags are exclusive, only one may be set */ TI_QBIND_FLAG_NSE =1< Date: Sat, 1 Nov 2025 15:24:14 +0100 Subject: [PATCH 02/23] Added type_all() function --- inc/doc.h | 1 + inc/ti/fn/fnmodtype.h | 57 ++------------------------ inc/ti/fn/fntypeall.h | 38 ++++++++++++++++++ inc/ti/type.h | 2 + inc/ti/version.h | 2 +- itest/test_collection_functions.py | 38 ++++++++++++++++++ src/ti/export.c | 6 +-- src/ti/qbind.c | 64 +++++++++++++++--------------- src/ti/type.c | 49 +++++++++++++++++++++++ 9 files changed, 169 insertions(+), 88 deletions(-) create mode 100644 inc/ti/fn/fntypeall.h diff --git a/inc/doc.h b/inc/doc.h index 7233e108..ba681849 100644 --- a/inc/doc.h +++ b/inc/doc.h @@ -126,6 +126,7 @@ #define DOC_TIMEVAL DOC_SEE("collection-api/timeval") #define DOC_TRY DOC_SEE("collection-api/try") #define DOC_TYPE DOC_SEE("collection-api/type") +#define DOC_TYPE_ALL DOC_SEE("collection-api/type_all") #define DOC_TYPE_ASSERT DOC_SEE("collection-api/type_assert") #define DOC_TYPE_COUNT DOC_SEE("collection-api/type_count") #define DOC_TYPE_INFO DOC_SEE("collection-api/type_info") diff --git a/inc/ti/fn/fnmodtype.h b/inc/ti/fn/fnmodtype.h index c5af1ca8..4887dd6d 100644 --- a/inc/ti/fn/fnmodtype.h +++ b/inc/ti/fn/fnmodtype.h @@ -44,24 +44,6 @@ static inline int modtype__delv_cb(ti_thing_t * thing, ti_field_t * field) return 0; } -typedef struct -{ - ti_type_t * type; - imap_t * imap; -} modtype__collect_t; - -static int modtype__collect_cb(ti_thing_t * thing, modtype__collect_t * w) -{ - if (thing->type_id == w->type->type_id) - { - if (imap_add(w->imap, ti_thing_key(thing), thing)) - return -1; - - ti_incref(thing); - } - return 0; -} - typedef struct { ti_field_t * field; @@ -311,37 +293,6 @@ static int modtype__all_cb(ti_thing_t * thing, modtype__all_t * w) return w->e->nr; } -static imap_t * modtype__collect_things(ti_query_t * query, ti_type_t * type) -{ - modtype__collect_t collect = { - .imap = imap_create(), - .type = type, - }; - - if (!collect.imap) - return NULL; - - if (ti_query_vars_walk( - query->vars, - query->collection, - (imap_cb) modtype__collect_cb, - &collect) || - imap_walk( - query->collection->things, - (imap_cb) modtype__collect_cb, - &collect) || - ti_gc_walk( - query->collection->gc, - (queue_cb) modtype__collect_cb, - &collect)) - { - imap_destroy(collect.imap, (imap_destroy_cb) ti_val_unsafe_drop); - return NULL; - } - - return collect.imap; -} - static void type__add( ti_query_t * query, ti_type_t * type, @@ -609,7 +560,7 @@ static void type__add( .e = e, }; - imap = modtype__collect_things(query, type); + imap = ti_type_collect_things(query, type); if (!imap) { ex_set_mem(e); @@ -775,7 +726,7 @@ static int type__mod_using_callback( if (!modjob.dval) goto fail2; /* error is set */ - imap = modtype__collect_things(query, field->type); + imap = ti_type_collect_things(query, field->type); if (!imap) { ex_set_mem(e); @@ -817,7 +768,7 @@ static int type__mod_using_callback( (imap_cb) modtype__unlocked_cb, NULL); - imap_after = modtype__collect_things(query, field->type); + imap_after = ti_type_collect_things(query, field->type); if (imap_after) { imap_difference_inplace(imap_after, imap); @@ -1393,7 +1344,7 @@ static void type__all( ti_closure_inc(closure, query, e)) goto fail0; - imap = modtype__collect_things(query, type); + imap = ti_type_collect_things(query, type); if (!imap) { ex_set_mem(e); diff --git a/inc/ti/fn/fntypeall.h b/inc/ti/fn/fntypeall.h new file mode 100644 index 00000000..3b890513 --- /dev/null +++ b/inc/ti/fn/fntypeall.h @@ -0,0 +1,38 @@ +#include + +static int do__f_type_all(ti_query_t * query, cleri_node_t * nd, ex_t * e) +{ + const int nargs = fn_get_nargs(nd); + imap_t * imap; + ti_type_t * type; + ti_vset_t * vset; + + if (fn_not_collection_scope("type_all", query, e) || + fn_nargs("type_all", DOC_TYPE_ALL, 1, nargs, e) || + ti_do_statement(query, nd->children, e) || + fn_arg_str("type_all", DOC_TYPE_ALL, 1, query->rval, e)) + return e->nr; + + type = ti_types_by_raw(query->collection->types, (ti_raw_t *) query->rval); + if (!type) + return ti_raw_err_not_found((ti_raw_t *) query->rval, "type", e); + + imap = ti_type_collect_things(query, type); + if (!imap) + { + ex_set_mem(e); + return e->nr; + } + + vset = ti_vset_create_imap(imap); + if (!vset) + { + ex_set_mem(e); + imap_destroy(imap, (imap_destroy_cb) ti_val_unsafe_drop); + return e->nr; + } + + ti_val_unsafe_drop(query->rval); + query->rval = (ti_val_t *) vset; + return e->nr; +} diff --git a/inc/ti/type.h b/inc/ti/type.h index 3feb3f6b..a7ed41fc 100644 --- a/inc/ti/type.h +++ b/inc/ti/type.h @@ -18,6 +18,7 @@ #include #include #include +#include ti_type_t * ti_type_create( ti_types_t * types, @@ -31,6 +32,7 @@ ti_type_t * ti_type_create_unnamed( ti_types_t * types, ti_raw_t * name, uint8_t flags); +imap_t * ti_type_collect_things(ti_query_t * query, ti_type_t * type); void ti_type_drop(ti_type_t * type); void ti_type_drop_unnamed(ti_type_t * type); void ti_type_del(ti_type_t * type, vec_t * vars); diff --git a/inc/ti/version.h b/inc/ti/version.h index 7c1d9774..a3025d8a 100644 --- a/inc/ti/version.h +++ b/inc/ti/version.h @@ -25,7 +25,7 @@ * "-rc0" * "" */ -#define TI_VERSION_PRE_RELEASE "-alpha5" +#define TI_VERSION_PRE_RELEASE "-alpha6" #define TI_MAINTAINER \ "Jeroen van der Heijden " diff --git a/itest/test_collection_functions.py b/itest/test_collection_functions.py index 4911ae16..37d71d68 100755 --- a/itest/test_collection_functions.py +++ b/itest/test_collection_functions.py @@ -6006,6 +6006,44 @@ async def test_bit_count(self, client): """) self.assertEqual(res, 42) + async def test_type_all(self, client): + q = client.query + + with self.assertRaisesRegex( + LookupError, + 'function `type_all` is undefined in the `@thingsdb` scope;'): + await q('type_all("T");', scope='/t') + + with self.assertRaisesRegex( + NumArgumentsError, + 'function `type_all` takes 1 argument but 0 were given;'): + await q('type_all();') + + with self.assertRaisesRegex( + TypeError, + 'function `type_all` expects argument 1 to be of type `str` ' + 'but got type `int` instead'): + await q('type_all(1);') + + with self.assertRaisesRegex( + LookupError, + 'type `T` not found'): + await q('type_all("T");') + + r = await q("""//ti + set_type('T', { + name: 'str', + }); + mod_type('T', 'add', 't', 'T?'); + .t1 = T{}; + .t2 = T{}; + .t2.t = .t2; + .t2 = nil; // test gc + t3 = T{}; + type_all('T').len(); + """) + self.assertEqual(r, 3) + if __name__ == '__main__': run_test(TestCollectionFunctions()) diff --git a/src/ti/export.c b/src/ti/export.c index 010ab054..ba3e4c8e 100644 --- a/src/ti/export.c +++ b/src/ti/export.c @@ -414,13 +414,13 @@ static int export__val(ti_fmt_t * fmt, ti_val_t * val) } case TI_VAL_SET: { - imap_t * map = VSET(val); - if (!map->n) + imap_t * imap = VSET(val); + if (!imap->n) return buf_append_str(buf, "set()"); if (buf_append_str(buf, "set(\n")) return -1; fmt->indent++; - if (imap_walk(map, (imap_cb) export__set_val, fmt)) + if (imap_walk(imap, (imap_cb) export__set_val, fmt)) return -1; fmt->indent--; return -( diff --git a/src/ti/qbind.c b/src/ti/qbind.c index 0752f910..c146c494 100644 --- a/src/ti/qbind.c +++ b/src/ti/qbind.c @@ -250,6 +250,7 @@ #include #include #include +#include #include #include #include @@ -301,11 +302,11 @@ static void qbind__statement(ti_qbind_t * qbind, cleri_node_t * nd); */ enum { - TOTAL_KEYWORDS = 276, + TOTAL_KEYWORDS = 277, MIN_WORD_LENGTH = 2, MAX_WORD_LENGTH = 17, - MIN_HASH_VALUE = 26, - MAX_HASH_VALUE = 710 + MIN_HASH_VALUE = 12, + MAX_HASH_VALUE = 751 }; /* @@ -317,32 +318,32 @@ static inline unsigned int qbind__hash( { static unsigned short asso_values[] = { - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 10, 10, - 10, 711, 10, 711, 9, 711, 12, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 9, 711, 14, 27, 102, - 17, 11, 128, 328, 77, 9, 24, 53, 11, 50, - 14, 43, 151, 45, 10, 9, 10, 60, 135, 248, - 156, 150, 30, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711, 711, 711, 711, 711, - 711, 711, 711, 711, 711, 711 + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 4, 3, + 6, 752, 3, 752, 3, 752, 4, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 2, 752, 19, 120, 85, + 23, 4, 136, 275, 180, 2, 6, 71, 24, 59, + 11, 41, 110, 44, 5, 2, 3, 28, 281, 191, + 227, 252, 25, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, + 752, 752, 752, 752, 752, 752 }; register unsigned int hval = n; @@ -732,15 +733,16 @@ qbind__fmap_t qbind__fn_mapping[TOTAL_KEYWORDS] = { {.name="to_thing", .fn=do__f_to_thing, CHAIN_CE_X}, {.name="to_type", .fn=do__f_to_type, CHAIN_CE_X}, {.name="to", .fn=do__f_to, CHAIN_NE}, + {.name="trim", .fn=do__f_trim, CHAIN_NE}, {.name="trim_left", .fn=do__f_trim_left, CHAIN_NE}, {.name="trim_right", .fn=do__f_trim_right, CHAIN_NE}, - {.name="trim", .fn=do__f_trim, CHAIN_NE}, {.name="try", .fn=do__f_try, XROOT_NE}, + {.name="type", .fn=do__f_type, ROOT_NE}, + {.name="type_all", .fn=do__f_type_all, ROOT_NE}, {.name="type_assert", .fn=do__f_type_assert, ROOT_NE}, {.name="type_count", .fn=do__f_type_count, ROOT_NE}, {.name="type_err", .fn=do__f_type_err, ROOT_NE}, {.name="type_info", .fn=do__f_type_info, ROOT_NE}, - {.name="type", .fn=do__f_type, ROOT_NE}, {.name="types_info", .fn=do__f_types_info, ROOT_NE}, {.name="unique", .fn=do__f_unique, CHAIN_NE}, {.name="unshift", .fn=do__f_unshift, CHAIN_CE_XX}, diff --git a/src/ti/type.c b/src/ti/type.c index 382a9312..fed4615b 100644 --- a/src/ti/type.c +++ b/src/ti/type.c @@ -26,6 +26,24 @@ #include #include +typedef struct +{ + ti_type_t * type; + imap_t * imap; +} type__collect_t; + +static int type__collect_cb(ti_thing_t * thing, type__collect_t * w) +{ + if (thing->type_id == w->type->type_id) + { + if (imap_add(w->imap, ti_thing_key(thing), thing)) + return -1; + + ti_incref(thing); + } + return 0; +} + static char * type__wrap_name(const char * name, size_t n) { char * wname; @@ -114,6 +132,37 @@ ti_type_t * ti_type_create_unnamed( return type; } +imap_t * ti_type_collect_things(ti_query_t * query, ti_type_t * type) +{ + type__collect_t collect = { + .imap = imap_create(), + .type = type, + }; + + if (!collect.imap) + return NULL; + + if (ti_query_vars_walk( + query->vars, + query->collection, + (imap_cb) type__collect_cb, + &collect) || + imap_walk( + query->collection->things, + (imap_cb) type__collect_cb, + &collect) || + ti_gc_walk( + query->collection->gc, + (queue_cb) type__collect_cb, + &collect)) + { + imap_destroy(collect.imap, (imap_destroy_cb) ti_val_unsafe_drop); + return NULL; + } + + return collect.imap; +} + /* used as a callback function and removes all cached type mappings */ static int type__map_cleanup(ti_type_t * t_haystack, ti_type_t * t_needle) { From 6aeb4a260b2eb2e4c7eefba435d39191ac17408c Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Sat, 1 Nov 2025 15:53:21 +0100 Subject: [PATCH 03/23] Added type_all() function --- inc/ti/fn/fntypeall.h | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/inc/ti/fn/fntypeall.h b/inc/ti/fn/fntypeall.h index 3b890513..fe0fb91d 100644 --- a/inc/ti/fn/fntypeall.h +++ b/inc/ti/fn/fntypeall.h @@ -3,7 +3,6 @@ static int do__f_type_all(ti_query_t * query, cleri_node_t * nd, ex_t * e) { const int nargs = fn_get_nargs(nd); - imap_t * imap; ti_type_t * type; ti_vset_t * vset; @@ -17,19 +16,31 @@ static int do__f_type_all(ti_query_t * query, cleri_node_t * nd, ex_t * e) if (!type) return ti_raw_err_not_found((ti_raw_t *) query->rval, "type", e); - imap = ti_type_collect_things(query, type); - if (!imap) + if (ti_type_is_wrap_only(type)) { - ex_set_mem(e); - return e->nr; + vset = ti_vset_create(); + if (!vset) + { + ex_set_mem(e); + return e->nr; + } } - - vset = ti_vset_create_imap(imap); - if (!vset) + else { - ex_set_mem(e); - imap_destroy(imap, (imap_destroy_cb) ti_val_unsafe_drop); - return e->nr; + imap_t * imap = ti_type_collect_things(query, type); + if (!imap) + { + ex_set_mem(e); + return e->nr; + } + + vset = ti_vset_create_imap(imap); + if (!vset) + { + ex_set_mem(e); + imap_destroy(imap, (imap_destroy_cb) ti_val_unsafe_drop); + return e->nr; + } } ti_val_unsafe_drop(query->rval); From cc042e7204719d651719e48d20c134b428e72666 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Sun, 2 Nov 2025 11:07:15 +0100 Subject: [PATCH 04/23] work in commit mig --- inc/ti.h | 1 + inc/ti/collection.t.h | 6 ++- inc/ti/mig.h | 41 +++++++++++++++++++ inc/ti/query.h | 1 + inc/ti/query.inline.h | 7 ++++ inc/ti/query.t.h | 7 ++-- inc/ti/task.h | 7 ++++ inc/ti/task.t.h | 3 ++ src/ti.c | 1 + src/ti/mig.c | 94 +++++++++++++++++++++++++++++++++++++++++++ src/ti/query.c | 32 ++++++++++++++- src/ti/task.c | 14 +++++++ src/ti/ttask.c | 44 ++++++++++++++++++++ 13 files changed, 251 insertions(+), 7 deletions(-) create mode 100644 inc/ti/mig.h create mode 100644 src/ti/mig.c diff --git a/inc/ti.h b/inc/ti.h index 184bfee9..81bfbbb2 100644 --- a/inc/ti.h +++ b/inc/ti.h @@ -107,6 +107,7 @@ struct ti_s vec_t * users; /* ti_user_t */ vec_t * access_node; /* ti_auth_t */ vec_t * access_thingsdb; /* ti_auth_t */ + vec_t * migs; /* ti_mig_t */ smap_t * procedures; /* ti_procedure_t */ smap_t * names; /* weak map for ti_name_t */ smap_t * qcache; /* pointer to cache in stack */ diff --git a/inc/ti/collection.t.h b/inc/ti/collection.t.h index 27357b9a..299efe51 100644 --- a/inc/ti/collection.t.h +++ b/inc/ti/collection.t.h @@ -8,11 +8,12 @@ typedef struct ti_collection_s ti_collection_t; #include #include -#include #include +#include +#include #include -#include #include +#include #include #include #include @@ -39,6 +40,7 @@ struct ti_collection_s uv_mutex_t * lock; /* only for watch/ unwatch/ away-mode */ vec_t * futures; /* no reference, type: ti_future_t */ vec_t * vtasks; /* tasks, type: ti_vtask_t */ + vec_t * migs; /* migration changes ti_mig_t */ guid_t guid; /* derived from collection->id */ }; diff --git a/inc/ti/mig.h b/inc/ti/mig.h new file mode 100644 index 00000000..f7ea0ee4 --- /dev/null +++ b/inc/ti/mig.h @@ -0,0 +1,41 @@ +/* + * ti/mig.h + */ +#ifndef TI_MIG_H_ +#define TI_MIG_H_ + +#include +#include +#include +#include +#include +#include + +typedef struct ti_mig_s ti_mig_t; + +ti_mig_t * ti_mig_create( + uint64_t id, + time_t ts, + const char * query, + size_t query_n, + const char * info, + size_t info_n, + const char * by, + size_t by_n); +ti_mig_t * ti_mig_create_q( + uint64_t id, + const char * query, + ti_raw_t * by); +void ti_mig_destroy(ti_mig_t * mig); +int ti_mig_keep_history(vec_t ** migs, _Bool state); + +struct ti_mig_s +{ + uint64_t id; /* equal to the change id */ + time_t ts; /* timestamp of the change */ + ti_raw_t * query; /* original query string */ + ti_raw_t * info; /* may be NULL */ + ti_raw_t * by; /* may be NULL */ +} + +#endif /* TI_MIG_H_ */ diff --git a/inc/ti/query.h b/inc/ti/query.h index e030cec2..a4072c9d 100644 --- a/inc/ti/query.h +++ b/inc/ti/query.h @@ -42,6 +42,7 @@ void ti_query_done(ti_query_t * query, ex_t * e, ti_query_done_cb cb); void ti_query_on_future_result(ti_future_t * future, ex_t * e); int ti_query_unpack_args(ti_query_t * query, mp_unp_t * up, ex_t * e); int ti_query_apply_scope(ti_query_t * query, ti_scope_t * scope, ex_t * e); +int ti_query_set_mig(ti_query_t * query); ti_prop_t * ti_query_var_get(ti_query_t * query, ti_name_t * name); ti_thing_t * ti_query_thing_from_id( ti_query_t * query, diff --git a/inc/ti/query.inline.h b/inc/ti/query.inline.h index 721255c5..a3087448 100644 --- a/inc/ti/query.inline.h +++ b/inc/ti/query.inline.h @@ -72,6 +72,13 @@ static inline uint64_t ti_query_scope_id(ti_query_t * query) : TI_SCOPE_NODE; } +static inline vec_t ** ti_query_migs(ti_query_t * query) +{ + return query->collection + ? &query->collection->migs + : &ti.migs; +} + static inline int ti_query_test_vset_operation(ti_query_t * query, ex_t * e) { if (!query->change && ti_vset_is_stored((ti_vset_t *) query->rval)) diff --git a/inc/ti/query.t.h b/inc/ti/query.t.h index 42920447..c5c8d79f 100644 --- a/inc/ti/query.t.h +++ b/inc/ti/query.t.h @@ -9,16 +9,17 @@ typedef int (*ti_query_vars_walk_cb)(void * data, void * arg); #include #include +#include #include #include -#include #include #include +#include #include #include -#include #include #include +#include #include #include @@ -72,7 +73,6 @@ typedef union ti_vtask_t * vtask; /* when called as task */ } ti_query_with_t; - struct ti_query_s { uint32_t local_stack; /* variable scopes start here */ @@ -92,6 +92,7 @@ struct ti_query_s ti_change_t * change; /* with reference, only when a change is required otherwise NULL */ + ti_mig_t * mig; /* NULL if no migration change */ vec_t * immutable_cache; /* ti_val_t, Only for immutable and collection independent variable and temporary used for procedures to populate the closure arguments diff --git a/inc/ti/task.h b/inc/ti/task.h index eb4022a9..5de1acb9 100644 --- a/inc/ti/task.h +++ b/inc/ti/task.h @@ -188,5 +188,12 @@ int ti_task_add_whitelist_del( ti_user_t * user, ti_val_t * val, int wid); +void ti_task_pack_mig_add(msgpack_packer * pk, ti_mig_t * mig); + +static inline size_t ti_task_size_mig_add(ti_mig_t * mig) +{ + return mig ? (48 + mig->query->n + mig->info->n + mig->by->n) : 0; +} + #endif /* TI_TASK_H_ */ diff --git a/inc/ti/task.t.h b/inc/ti/task.t.h index 8de001b8..4f353607 100644 --- a/inc/ti/task.t.h +++ b/inc/ti/task.t.h @@ -92,6 +92,9 @@ typedef enum TI_TASK_ROOM_SET_NAME, /* 79 */ TI_TASK_WHITELIST_ADD, /* 80 */ TI_TASK_WHITELIST_DEL, /* 81 */ + TI_TASK_KEEP_HISTORY, /* 82 */ + TI_TASK_MIG_ADD, /* 83 */ + TI_TASK_MIG_DEL, /* 84 */ } ti_task_enum; typedef struct ti_task_s ti_task_t; diff --git a/src/ti.c b/src/ti.c index c52739d2..3378f782 100644 --- a/src/ti.c +++ b/src/ti.c @@ -89,6 +89,7 @@ int ti_create(void) ti.node = NULL; ti.store = NULL; ti.tasks = NULL; + ti.migs = NULL; ti.access_node = vec_new(0); ti.access_thingsdb = vec_new(0); ti.procedures = smap_create(); diff --git a/src/ti/mig.c b/src/ti/mig.c new file mode 100644 index 00000000..545df0c5 --- /dev/null +++ b/src/ti/mig.c @@ -0,0 +1,94 @@ +/* + * ti/mig.c + */ +#include +#include +#include +#include + +ti_mig_t * ti_mig_create( + uint64_t id, + time_t ts, + const char * query, + size_t query_n, + const char * info, + size_t info_n, + const char * by, + size_t by_n) +{ + ti_mig_t * mig = malloc(sizeof(ti_mig_t)); + if (!mig) + return NULL; + + mig->id = id; + mig->ts = ts; + + mig->query = ti_str_create(query, query_n); + if (!mig->query) + goto fail0; + + mig->info = ti_str_create(info, info_n); + if (!mig->info) + goto fail1; + + mig->by = ti_str_create(by, by_n); + if (!mig->by) + goto fail2; + + return mig; +fail2: + ti_val_unsafe_drop(mig->info); +fail1: + ti_val_unsafe_drop(mig->query); +fail0: + free(mig); +} + +ti_mig_t * ti_mig_create_q( + uint64_t id, + const char * query, + ti_raw_t * by) +{ + ti_mig_t * mig = malloc(sizeof(ti_mig_t)); + if (!mig) + return NULL; + + mig->id = id; + mig->ts = (time_t) util_now_usec(); + + mig->query = ti_str_from_str(query); + if (!mig->query) + { + free(mig); + return NULL; + } + + mig->info = ti_val_empty_str(); + mig->by = ti_grab(by); + return mig; +} + +void ti_mig_destroy(ti_mig_t * mig) +{ + if (!mig) + return; + ti_val_unsafe_drop(mig->query); + ti_val_unsafe_drop(mig->info); + ti_val_unsafe_drop(mig->by); + free(mig); +} + +int ti_mig_keep_history(vec_t ** migs, _Bool state) +{ + if (state) + { + if (*migs) + return 0; + + *migs = vec_new(8); + return !!(*migs); + } + vec_destroy(*migs, ti_mig_destroy); + *migs = NULL; + return 0; +} \ No newline at end of file diff --git a/src/ti/query.c b/src/ti/query.c index 62233935..587f9264 100644 --- a/src/ti/query.c +++ b/src/ti/query.c @@ -62,7 +62,7 @@ ti_query_run_cb ti_query_run_map[] = { */ static ti_cpkg_t * query__cpkg_change(ti_query_t * query) { - size_t init_buffer_sz = 40; + size_t init_buffer_sz = 40 + ti_task_size_mig_add(query->mig); msgpack_packer pk; msgpack_sbuffer buffer; ti_cpkg_t * cpkg; @@ -76,7 +76,7 @@ static ti_cpkg_t * query__cpkg_change(ti_query_t * query) return NULL; msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); - msgpack_pack_array(&pk, tasks->n+2); + msgpack_pack_array(&pk, tasks->n + 2 + !!query->mig); msgpack_pack_uint64(&pk, query->change->id); msgpack_pack_uint64(&pk, tasks->n && query->collection ? query->collection->id @@ -90,6 +90,9 @@ static ti_cpkg_t * query__cpkg_change(ti_query_t * query) mp_pack_append(&pk, data->data, data->n); } + if (query->mig) + ti_task_pack_mig_add(&pk, query->mig); + pkg = (ti_pkg_t *) buffer.data; pkg_init(pkg, 0, TI_PROTO_NODE_CHANGE, buffer.size); @@ -157,6 +160,31 @@ int ti_query_apply_scope(ti_query_t * query, ti_scope_t * scope, ex_t * e) return e->nr; } +int ti_query_check_commit + +int ti_query_set_mig(ti_query_t * query) +{ + if (query->mig) + return 0; + + if (!query->user || + !query->change || + query->with_tp != TI_QUERY_WITH_PARSERES) + return -1; + + query->mig = ti_mig_create_q( + query->change->id, + (time_t) util_now_usec(), + query->with.parseres->str, + strlen(query->with.parseres->str), + "", + 0, + query->user->name->data, + query->user->name->n); + + return !!query->mig; +} + ti_query_t * ti_query_create(uint8_t flags) { ti_query_t * query = calloc(1, sizeof(ti_query_t)); diff --git a/src/ti/task.c b/src/ti/task.c index a4d979f5..7d206070 100644 --- a/src/ti/task.c +++ b/src/ti/task.c @@ -3138,4 +3138,18 @@ int ti_task_add_whitelist_del( fail_pack: msgpack_sbuffer_destroy(&buffer); return -1; +} + +void ti_task_pack_mig_add(msgpack_packer * pk, ti_mig_t * mig) +{ + msgpack_pack_array(pk, 2); + + msgpack_pack_uint8(pk, TI_TASK_MIG_ADD); + msgpack_pack_array(pk, 5); + + msgpack_pack_uint64(pk, mig->id); + msgpack_pack_uint64(pk, (uint64_t) mig->ts); + mp_pack_strn(pk, mig->query->data, mig->query->n); + mp_pack_strn(pk, mig->info->data, mig->info->n); + mp_pack_strn(pk, mig->by->data, mig->by->n); } \ No newline at end of file diff --git a/src/ti/ttask.c b/src/ti/ttask.c index 51dc1808..94065c03 100644 --- a/src/ti/ttask.c +++ b/src/ti/ttask.c @@ -703,6 +703,47 @@ static int ttask__whitelist_del(mp_unp_t * up) return 0; } +static int ttask__keep_historyl(mp_unp_t * up) +{ + vec_t ** migs; + mp_obj_t obj, mp_scope, mp_state; + + if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 2 + mp_next(up, &mp_scope) != MP_U64 || + mp_next(up, &mp_state) != MP_BOOL) + { + log_critical("task `keep_history`: invalid task data"); + return -1; + } + + if (mp_scope.via.u64 > 1) + { + ti_collection_t * collection = \ + ti_collections_get_by_id(mp_scope.via.u64); + + if (!collection) + { + log_critical( + "task `keep_history`: "TI_COLLECTION_ID" not found", + mp_scope.via.u64); + return -1; + } + migs = &collection->migs; + } + else + { + migs = &ti.migs; + } + + if (ti_mig_keep_history(&migs, mp_state.via.bool_)) + { + log_critical("failed to set `keep_history`"); + return -1; + } + + return 0; +} + /* * Returns 0 on success * - for example: {'id': id, 'key': value}, 'expire_ts': ts, 'description':..} @@ -1735,6 +1776,9 @@ int ti_ttask_run(ti_change_t * change, mp_unp_t * up) case TI_TASK_ROOM_SET_NAME: break; case TI_TASK_WHITELIST_ADD: return ttask__whitelist_add(up); case TI_TASK_WHITELIST_DEL: return ttask__whitelist_del(up); + case TI_TASK_KEEP_HISTORY: return ttask__keep_history(up); + case TI_TASK_MIG_ADD: return ttask__mig_add(up); + case TI_TASK_MIG_DEL: return ttask__mig_del(up); } log_critical("unknown thingsdb task: %"PRIu64, mp_task.via.u64); From 4c1ebeb733aa8a816f4eaa2850bf1dcb1ba1352f Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Sun, 2 Nov 2025 22:27:33 +0100 Subject: [PATCH 05/23] work on history --- inc/doc.h | 4 ++- inc/ti/fn/fn.h | 12 +++++++ inc/ti/fn/fncommit.h | 77 +++++++++++++++++++++++++++++++++++++++++++ inc/ti/fn/fnnewtype.h | 1 + inc/ti/fn/fnsettype.h | 1 + inc/ti/mig.h | 11 ++++--- inc/ti/query.h | 1 - inc/ti/task.h | 8 +---- inc/ti/task.t.h | 2 +- src/ti/mig.c | 19 +++++++++-- src/ti/query.c | 49 ++++++++++----------------- src/ti/task.c | 27 +++++++++++++-- src/ti/ttask.c | 12 +++---- 13 files changed, 169 insertions(+), 55 deletions(-) create mode 100644 inc/ti/fn/fncommit.h diff --git a/inc/doc.h b/inc/doc.h index ba681849..0db04195 100644 --- a/inc/doc.h +++ b/inc/doc.h @@ -20,14 +20,15 @@ #define DOC_BYTES DOC_SEE("collection-api/bytes") #define DOC_CHANGE_ID DOC_SEE("collection-api/change_id") #define DOC_CLOSURE DOC_SEE("collection-api/closure") +#define DOC_COMMIT DOC_SEE("collection-api/commit") #define DOC_DATETIME DOC_SEE("collection-api/datetime") #define DOC_DEEP DOC_SEE("collection-api/deep") #define DOC_DEL_ENUM DOC_SEE("collection-api/del_enum") #define DOC_DEL_TYPE DOC_SEE("collection-api/del_type") #define DOC_ENUM DOC_SEE("collection-api/enum") +#define DOC_ENUMS_INFO DOC_SEE("collection-api/enums_info") #define DOC_ENUM_INFO DOC_SEE("collection-api/enum_info") #define DOC_ENUM_MAP DOC_SEE("collection-api/enum_map") -#define DOC_ENUMS_INFO DOC_SEE("collection-api/enums_info") #define DOC_ERR DOC_SEE("collection-api/err") #define DOC_EXPORT DOC_SEE("collection-api/export") #define DOC_FLOAT DOC_SEE("collection-api/float") @@ -178,6 +179,7 @@ #define DOC_RESTORE DOC_SEE("thingsdb-api/restore") #define DOC_REVOKE DOC_SEE("thingsdb-api/revoke") #define DOC_SET_DEFAULT_DEEP DOC_SEE("thingsdb-api/set_default_deep") +#define DOC_SET_HISTORY DOC_SEE("thingsdb-api/set_history") #define DOC_SET_MODULE_CONF DOC_SEE("thingsdb-api/set_module_conf") #define DOC_SET_MODULE_SCOPE DOC_SEE("thingsdb-api/set_module_scope") #define DOC_SET_PASSWORD DOC_SEE("thingsdb-api/set_password") diff --git a/inc/ti/fn/fn.h b/inc/ti/fn/fn.h index 5bfdad0c..b6b1dd74 100644 --- a/inc/ti/fn/fn.h +++ b/inc/ti/fn/fn.h @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -443,6 +444,17 @@ static inline int fn_arg_name_check( return e->nr; } +static int fn_commit(const char * name, ti_query_t * query, ex_t * e) +{ + if (ti_query_migs(query) && !query->mig) + ex_set(e, EX_OPERATION, + "function `%s` requires a commit " + "before it can be used in the `%s` scope"DOC_COMMIT, + name, + ti_query_scope_name(query)); + return e->nr; +} + static inline int fn_not_collection_scope( const char * name, ti_query_t * query, diff --git a/inc/ti/fn/fncommit.h b/inc/ti/fn/fncommit.h new file mode 100644 index 00000000..f9a14ead --- /dev/null +++ b/inc/ti/fn/fncommit.h @@ -0,0 +1,77 @@ +#include + +static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) +{ + const int nargs = fn_get_nargs(nd); + vec_t ** migs = ti_query_migs(query); + ti_mig_t * mig; + + if (fn_not_thingsdb_or_collection_scope("commit", query, e) || + fn_nargs("commit", DOC_COMMIT, 1, nargs, e) || + ti_do_statement(query, nd->children, e) || + fn_arg_str("commit", DOC_COMMIT, 1, query->rval, e)) + return e->nr; + + raw = (ti_raw_t *) query->rval; + query->rval = NULL; + + if (!(*migs)) + { + ex_set(e, EX_OPERATION, + "commit history is not enabled for the `%s` scope" + DOC_SET_HISTORY, + ti_query_scope_name(query)); + goto fail0; + } + + if (!query->change) + { + ex_set(e, EX_OPERATION, "commit without a change"DOC_SET_HISTORY); + goto fail0; + } + + if (!raw->n) + { + ex_set(e, EX_VALUE_ERROR, "commit message must not be empty"); + goto fail0; + } + + if (query->mig) + { + ex_set(e, EX_OPERATION, "commit message already set"); + goto fail0; + } + + switch ((ti_query_with_enum) query->with_tp) + { + case TI_QUERY_WITH_PARSERES: break; + case TI_QUERY_WITH_PROCEDURE: + ex_set(e, EX_OPERATION, + "function `commit` is not allowed in a procedure"); + goto fail0; + case TI_QUERY_WITH_FUTURE: + ex_set(e, EX_OPERATION, + "function `commit` is not allowed in a future"); + goto fail0; + case TI_QUERY_WITH_TASK: + case TI_QUERY_WITH_TASK_FINISH: + ex_set(e, EX_OPERATION, + "function `commit` is not allowed in a task"); + goto fail0; + } + + mig = ti_mig_create_q( + query->change->id, query->with.parseres->str, query->user->name); + + if (!mig || vec_push(migs, mig)) + { + ti_mig_destroy(mig); + ex_set_mem(e); + goto fail0; + } + + query->rval = ti_nil_get(); +fail0: + ti_val_unsafe_drop((ti_val_t *) raw); + return e->nr; +} diff --git a/inc/ti/fn/fnnewtype.h b/inc/ti/fn/fnnewtype.h index 228d22ad..2089739a 100644 --- a/inc/ti/fn/fnnewtype.h +++ b/inc/ti/fn/fnnewtype.h @@ -11,6 +11,7 @@ static int do__f_new_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) uint8_t flags = 0; if (fn_not_collection_scope("new_type", query, e) || + fn_commit("new_type", query, e) || fn_nargs_range("new_type", DOC_NEW_TYPE, 1, 3, nargs, e) || ti_do_statement(query, child, e) || fn_arg_str("new_type", DOC_NEW_TYPE, 1, query->rval, e)) diff --git a/inc/ti/fn/fnsettype.h b/inc/ti/fn/fnsettype.h index 335ade99..58f5e7a1 100644 --- a/inc/ti/fn/fnsettype.h +++ b/inc/ti/fn/fnsettype.h @@ -14,6 +14,7 @@ static int do__f_set_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) _Bool hid = false; if (fn_not_collection_scope("set_type", query, e) || + fn_commit("set_type", query, e) || fn_nargs_range("set_type", DOC_SET_TYPE, 2, 4, nargs, e) || ti_do_statement(query, child, e) || fn_arg_str("set_type", DOC_SET_TYPE, 1, query->rval, e)) diff --git a/inc/ti/mig.h b/inc/ti/mig.h index f7ea0ee4..d110e996 100644 --- a/inc/ti/mig.h +++ b/inc/ti/mig.h @@ -21,21 +21,24 @@ ti_mig_t * ti_mig_create( const char * info, size_t info_n, const char * by, - size_t by_n); + size_t by_n, + const char * err_msg, /* may be NULL*/ + size_t err_msg_n); ti_mig_t * ti_mig_create_q( uint64_t id, const char * query, ti_raw_t * by); void ti_mig_destroy(ti_mig_t * mig); -int ti_mig_keep_history(vec_t ** migs, _Bool state); +int ti_mig_set_history(vec_t ** migs, _Bool state); struct ti_mig_s { uint64_t id; /* equal to the change id */ time_t ts; /* timestamp of the change */ ti_raw_t * query; /* original query string */ - ti_raw_t * info; /* may be NULL */ - ti_raw_t * by; /* may be NULL */ + ti_raw_t * info; /* never NULL */ + ti_raw_t * by; /* never NULL */ + ti_raw_t * err_msg; /* may be NULL */ } #endif /* TI_MIG_H_ */ diff --git a/inc/ti/query.h b/inc/ti/query.h index a4072c9d..e030cec2 100644 --- a/inc/ti/query.h +++ b/inc/ti/query.h @@ -42,7 +42,6 @@ void ti_query_done(ti_query_t * query, ex_t * e, ti_query_done_cb cb); void ti_query_on_future_result(ti_future_t * future, ex_t * e); int ti_query_unpack_args(ti_query_t * query, mp_unp_t * up, ex_t * e); int ti_query_apply_scope(ti_query_t * query, ti_scope_t * scope, ex_t * e); -int ti_query_set_mig(ti_query_t * query); ti_prop_t * ti_query_var_get(ti_query_t * query, ti_name_t * name); ti_thing_t * ti_query_thing_from_id( ti_query_t * query, diff --git a/inc/ti/task.h b/inc/ti/task.h index 5de1acb9..47aaae05 100644 --- a/inc/ti/task.h +++ b/inc/ti/task.h @@ -188,12 +188,6 @@ int ti_task_add_whitelist_del( ti_user_t * user, ti_val_t * val, int wid); -void ti_task_pack_mig_add(msgpack_packer * pk, ti_mig_t * mig); - -static inline size_t ti_task_size_mig_add(ti_mig_t * mig) -{ - return mig ? (48 + mig->query->n + mig->info->n + mig->by->n) : 0; -} - +int ti_task_add_mig_add(ti_task_t * task, ti_mig_t * mig); #endif /* TI_TASK_H_ */ diff --git a/inc/ti/task.t.h b/inc/ti/task.t.h index 4f353607..b2ed733a 100644 --- a/inc/ti/task.t.h +++ b/inc/ti/task.t.h @@ -92,7 +92,7 @@ typedef enum TI_TASK_ROOM_SET_NAME, /* 79 */ TI_TASK_WHITELIST_ADD, /* 80 */ TI_TASK_WHITELIST_DEL, /* 81 */ - TI_TASK_KEEP_HISTORY, /* 82 */ + TI_TASK_SET_HISTORY, /* 82 */ TI_TASK_MIG_ADD, /* 83 */ TI_TASK_MIG_DEL, /* 84 */ } ti_task_enum; diff --git a/src/ti/mig.c b/src/ti/mig.c index 545df0c5..ccef1717 100644 --- a/src/ti/mig.c +++ b/src/ti/mig.c @@ -14,7 +14,9 @@ ti_mig_t * ti_mig_create( const char * info, size_t info_n, const char * by, - size_t by_n) + size_t by_n, + const char * err_msg, + size_t err_msg_n) { ti_mig_t * mig = malloc(sizeof(ti_mig_t)); if (!mig) @@ -35,7 +37,18 @@ ti_mig_t * ti_mig_create( if (!mig->by) goto fail2; + if (err_msg) + { + mig->err_msg = ti_str_create(err_msg, err_msg_n); + if (!mig->err_msg) + goto fail3; + } + else + mig->err_msg = NULL; + return mig; +fail3: + ti_val_unsafe_drop(mig->by); fail2: ti_val_unsafe_drop(mig->info); fail1: @@ -65,6 +78,7 @@ ti_mig_t * ti_mig_create_q( mig->info = ti_val_empty_str(); mig->by = ti_grab(by); + mig->err_msg = NULL; return mig; } @@ -75,10 +89,11 @@ void ti_mig_destroy(ti_mig_t * mig) ti_val_unsafe_drop(mig->query); ti_val_unsafe_drop(mig->info); ti_val_unsafe_drop(mig->by); + ti_val_drop(mig->err_msg); free(mig); } -int ti_mig_keep_history(vec_t ** migs, _Bool state) +int ti_mig_set_history(vec_t ** migs, _Bool state) { if (state) { diff --git a/src/ti/query.c b/src/ti/query.c index 587f9264..99deb19c 100644 --- a/src/ti/query.c +++ b/src/ti/query.c @@ -62,7 +62,7 @@ ti_query_run_cb ti_query_run_map[] = { */ static ti_cpkg_t * query__cpkg_change(ti_query_t * query) { - size_t init_buffer_sz = 40 + ti_task_size_mig_add(query->mig); + size_t init_buffer_sz = 40; msgpack_packer pk; msgpack_sbuffer buffer; ti_cpkg_t * cpkg; @@ -76,7 +76,7 @@ static ti_cpkg_t * query__cpkg_change(ti_query_t * query) return NULL; msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); - msgpack_pack_array(&pk, tasks->n + 2 + !!query->mig); + msgpack_pack_array(&pk, tasks->n + 2); msgpack_pack_uint64(&pk, query->change->id); msgpack_pack_uint64(&pk, tasks->n && query->collection ? query->collection->id @@ -90,9 +90,6 @@ static ti_cpkg_t * query__cpkg_change(ti_query_t * query) mp_pack_append(&pk, data->data, data->n); } - if (query->mig) - ti_task_pack_mig_add(&pk, query->mig); - pkg = (ti_pkg_t *) buffer.data; pkg_init(pkg, 0, TI_PROTO_NODE_CHANGE, buffer.size); @@ -160,31 +157,6 @@ int ti_query_apply_scope(ti_query_t * query, ti_scope_t * scope, ex_t * e) return e->nr; } -int ti_query_check_commit - -int ti_query_set_mig(ti_query_t * query) -{ - if (query->mig) - return 0; - - if (!query->user || - !query->change || - query->with_tp != TI_QUERY_WITH_PARSERES) - return -1; - - query->mig = ti_mig_create_q( - query->change->id, - (time_t) util_now_usec(), - query->with.parseres->str, - strlen(query->with.parseres->str), - "", - 0, - query->user->name->data, - query->user->name->n); - - return !!query->mig; -} - ti_query_t * ti_query_create(uint8_t flags) { ti_query_t * query = calloc(1, sizeof(ti_query_t)); @@ -982,8 +954,23 @@ void ti_query_run_parseres(ti_query_t * query) stop: if (query->change) - query__change_handle(query); /* errors will be logged only */ + { + if (query->mig) + { + ti_task_t * task; + + if (e.nr) /* set error if any, not critical if failed */ + query->mig->err_msg = ti_str_create(e.msg, e.n); + task = ti_task_get_task( + query->change, + query->collection ? query->collection->root : ti.thing0); + + if (task) + (void) ti_task_add_mig_add(task, query->mig); + } + query__change_handle(query); /* errors will be logged only */ + } ti_query_done(query, &e, &ti_query_send_response); } diff --git a/src/ti/task.c b/src/ti/task.c index 7d206070..1c32a651 100644 --- a/src/ti/task.c +++ b/src/ti/task.c @@ -3140,16 +3140,39 @@ int ti_task_add_whitelist_del( return -1; } -void ti_task_pack_mig_add(msgpack_packer * pk, ti_mig_t * mig) +int ti_task_add_mig_add(ti_task_t * task, ti_mig_t * mig) { + size_t alloc = (50 + mig->query->n + mig->info->n + mig->by->n + ( + mig->err_msg ? mig->err_msg->n : 0) + ); + ti_data_t * data; + msgpack_packer pk; + msgpack_sbuffer buffer; + + if (mp_sbuffer_alloc_init(&buffer, alloc, sizeof(ti_data_t))) + return -1; + msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); + msgpack_pack_array(pk, 2); msgpack_pack_uint8(pk, TI_TASK_MIG_ADD); - msgpack_pack_array(pk, 5); + msgpack_pack_array(pk, 5 + !!mig->err_msg); msgpack_pack_uint64(pk, mig->id); msgpack_pack_uint64(pk, (uint64_t) mig->ts); mp_pack_strn(pk, mig->query->data, mig->query->n); mp_pack_strn(pk, mig->info->data, mig->info->n); mp_pack_strn(pk, mig->by->data, mig->by->n); + if (mig->err_msg) + mp_pack_strn(pk, mig->err_msg->data, mig->err_msg->n); + + if (vec_push(&task->list, data)) + goto fail_data; + + task__upd_approx_sz(task, data); + return 0; + +fail_data: + free(data); + return -1; } \ No newline at end of file diff --git a/src/ti/ttask.c b/src/ti/ttask.c index 94065c03..b42bf934 100644 --- a/src/ti/ttask.c +++ b/src/ti/ttask.c @@ -703,7 +703,7 @@ static int ttask__whitelist_del(mp_unp_t * up) return 0; } -static int ttask__keep_historyl(mp_unp_t * up) +static int ttask__set_historyl(mp_unp_t * up) { vec_t ** migs; mp_obj_t obj, mp_scope, mp_state; @@ -712,7 +712,7 @@ static int ttask__keep_historyl(mp_unp_t * up) mp_next(up, &mp_scope) != MP_U64 || mp_next(up, &mp_state) != MP_BOOL) { - log_critical("task `keep_history`: invalid task data"); + log_critical("task `set_history`: invalid task data"); return -1; } @@ -724,7 +724,7 @@ static int ttask__keep_historyl(mp_unp_t * up) if (!collection) { log_critical( - "task `keep_history`: "TI_COLLECTION_ID" not found", + "task `set_history`: "TI_COLLECTION_ID" not found", mp_scope.via.u64); return -1; } @@ -735,9 +735,9 @@ static int ttask__keep_historyl(mp_unp_t * up) migs = &ti.migs; } - if (ti_mig_keep_history(&migs, mp_state.via.bool_)) + if (ti_mig_set_history(&migs, mp_state.via.bool_)) { - log_critical("failed to set `keep_history`"); + log_critical("failed to set `set_history`"); return -1; } @@ -1776,7 +1776,7 @@ int ti_ttask_run(ti_change_t * change, mp_unp_t * up) case TI_TASK_ROOM_SET_NAME: break; case TI_TASK_WHITELIST_ADD: return ttask__whitelist_add(up); case TI_TASK_WHITELIST_DEL: return ttask__whitelist_del(up); - case TI_TASK_KEEP_HISTORY: return ttask__keep_history(up); + case TI_TASK_SET_HISTORY: return ttask__set_history(up); case TI_TASK_MIG_ADD: return ttask__mig_add(up); case TI_TASK_MIG_DEL: return ttask__mig_del(up); } From 4effc0baa948e5def9f3f9543a8f9bc374d341e7 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Mon, 3 Nov 2025 18:19:41 +0100 Subject: [PATCH 06/23] work on commit history --- inc/doc.h | 2 + inc/ti.h | 2 +- inc/ti/collection.inline.h | 9 ++- inc/ti/collection.t.h | 4 +- inc/ti/commit.h | 44 +++++++++++++++ inc/ti/commits.h | 21 +++++++ inc/ti/fn/fn.h | 4 +- inc/ti/fn/fncommit.h | 59 +++++++++++--------- inc/ti/fn/fnhistory.h | 37 +++++++++++++ inc/ti/fn/fnsethistory.h | 50 +++++++++++++++++ inc/ti/mig.h | 44 --------------- inc/ti/query.inline.h | 6 +- inc/ti/query.t.h | 4 +- inc/ti/task.h | 2 +- inc/ti/task.t.h | 4 +- src/ti.c | 14 +++-- src/ti/commit.c | 110 +++++++++++++++++++++++++++++++++++++ src/ti/commits.c | 59 ++++++++++++++++++++ src/ti/ctask.c | 67 ++++++++++++++++++++++ src/ti/export.c | 10 ++++ src/ti/mig.c | 109 ------------------------------------ src/ti/query.c | 6 +- src/ti/task.c | 24 ++++---- src/ti/ttask.c | 12 ++-- 24 files changed, 485 insertions(+), 218 deletions(-) create mode 100644 inc/ti/commit.h create mode 100644 inc/ti/commits.h create mode 100644 inc/ti/fn/fnhistory.h create mode 100644 inc/ti/fn/fnsethistory.h delete mode 100644 inc/ti/mig.h create mode 100644 src/ti/commit.c create mode 100644 src/ti/commits.c delete mode 100644 src/ti/mig.c diff --git a/inc/doc.h b/inc/doc.h index 0db04195..e1469a1a 100644 --- a/inc/doc.h +++ b/inc/doc.h @@ -154,6 +154,7 @@ #define DOC_COLLECTIONS_INFO DOC_SEE("thingsdb-api/collections_info") #define DOC_DEL_COLLECTION DOC_SEE("thingsdb-api/del_collection") #define DOC_DEL_EXPIRED DOC_SEE("thingsdb-api/del_expired") +#define DOC_DEL_HISTORY DOC_SEE("thingsdb-api/del_history") #define DOC_DEL_MODULE DOC_SEE("thingsdb-api/del_module") #define DOC_DEL_NODE DOC_SEE("thingsdb-api/del_node") #define DOC_DEL_TOKEN DOC_SEE("thingsdb-api/del_token") @@ -165,6 +166,7 @@ #define DOC_HAS_NODE DOC_SEE("thingsdb-api/has_node") #define DOC_HAS_TOKEN DOC_SEE("thingsdb-api/has_token") #define DOC_HAS_USER DOC_SEE("thingsdb-api/has_user") +#define DOC_HISTORY DOC_SEE("thingsdb-api/history") #define DOC_MODULE_INFO DOC_SEE("thingsdb-api/module_info") #define DOC_MODULES_INFO DOC_SEE("thingsdb-api/modules_info") #define DOC_NEW_COLLECTION DOC_SEE("thingsdb-api/new_collection") diff --git a/inc/ti.h b/inc/ti.h index 81bfbbb2..a80fffbe 100644 --- a/inc/ti.h +++ b/inc/ti.h @@ -107,7 +107,7 @@ struct ti_s vec_t * users; /* ti_user_t */ vec_t * access_node; /* ti_auth_t */ vec_t * access_thingsdb; /* ti_auth_t */ - vec_t * migs; /* ti_mig_t */ + vec_t * commits; /* ti_commit_t */ smap_t * procedures; /* ti_procedure_t */ smap_t * names; /* weak map for ti_name_t */ smap_t * qcache; /* pointer to cache in stack */ diff --git a/inc/ti/collection.inline.h b/inc/ti/collection.inline.h index 13a9cfc4..93dd38d9 100644 --- a/inc/ti/collection.inline.h +++ b/inc/ti/collection.inline.h @@ -13,7 +13,7 @@ static inline int ti_collection_to_pk( msgpack_packer * pk) { return -( - msgpack_pack_map(pk, 7) || + msgpack_pack_map(pk, 8) || mp_pack_str(pk, "collection_id") || msgpack_pack_uint64(pk, collection->id) || @@ -34,7 +34,12 @@ static inline int ti_collection_to_pk( msgpack_pack_uint64(pk, collection->deep) || mp_pack_str(pk, "next_free_id") || - msgpack_pack_uint64(pk, collection->next_free_id) + msgpack_pack_uint64(pk, collection->next_free_id) || + + mp_pack_str(pk, "commit_history") || + (collection->commits + ? msgpack_pack_uint32(pk, collection->commits->n) + : mp_pack_str(pk, "disabled")) ); } diff --git a/inc/ti/collection.t.h b/inc/ti/collection.t.h index 299efe51..cb1769ab 100644 --- a/inc/ti/collection.t.h +++ b/inc/ti/collection.t.h @@ -9,7 +9,7 @@ typedef struct ti_collection_s ti_collection_t; #include #include #include -#include +#include #include #include #include @@ -40,7 +40,7 @@ struct ti_collection_s uv_mutex_t * lock; /* only for watch/ unwatch/ away-mode */ vec_t * futures; /* no reference, type: ti_future_t */ vec_t * vtasks; /* tasks, type: ti_vtask_t */ - vec_t * migs; /* migration changes ti_mig_t */ + vec_t * commits; /* migration changes ti_commit_t */ guid_t guid; /* derived from collection->id */ }; diff --git a/inc/ti/commit.h b/inc/ti/commit.h new file mode 100644 index 00000000..0308cddc --- /dev/null +++ b/inc/ti/commit.h @@ -0,0 +1,44 @@ +/* + * ti/commit.h + */ +#ifndef TI_COMMIT_H_ +#define TI_COMMIT_H_ + +#include +#include +#include +#include +#include +#include + +typedef struct ti_commit_s ti_commit_t; + +ti_commit_t * ti_commit_create( + uint64_t id, + time_t ts, + const char * code, + size_t code_n, + const char * message, + size_t message_n, + const char * by, + size_t by_n, + const char * err_msg, /* may be NULL*/ + size_t err_msg_n); +ti_commit_t * ti_commit_make( + uint64_t id, + const char * code, + ti_raw_t * by, + ti_raw_t * message); +void ti_commit_destroy(ti_commit_t * commit); + +struct ti_commit_s +{ + uint64_t id; /* equal to the change id */ + time_t ts; /* timestamp of the change */ + ti_raw_t * code; /* original query string */ + ti_raw_t * message; /* never NULL */ + ti_raw_t * by; /* never NULL */ + ti_raw_t * err_msg; /* may be NULL */ +} + +#endif /* TI_COMMIT_H_ */ diff --git a/inc/ti/commits.h b/inc/ti/commits.h new file mode 100644 index 00000000..16661a1d --- /dev/null +++ b/inc/ti/commits.h @@ -0,0 +1,21 @@ +/* + * ti/commits.h + */ +#ifndef TI_COMMITS_H_ +#define TI_COMMITS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int ti_commits_set_history(vec_t ** commits, _Bool state); +ti_varr_t * ti_commits_find(vec_t ** commits, ti_thing_t * thing); +vec_t * ti_commits_del(vec_t ** commits, ti_thing_t * thing); + +#endif /* TI_COMMITS_H_ */ diff --git a/inc/ti/fn/fn.h b/inc/ti/fn/fn.h index b6b1dd74..424878ca 100644 --- a/inc/ti/fn/fn.h +++ b/inc/ti/fn/fn.h @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include @@ -446,7 +446,7 @@ static inline int fn_arg_name_check( static int fn_commit(const char * name, ti_query_t * query, ex_t * e) { - if (ti_query_migs(query) && !query->mig) + if (ti_query_commits(query) && !query->commit) ex_set(e, EX_OPERATION, "function `%s` requires a commit " "before it can be used in the `%s` scope"DOC_COMMIT, diff --git a/inc/ti/fn/fncommit.h b/inc/ti/fn/fncommit.h index f9a14ead..73748d85 100644 --- a/inc/ti/fn/fncommit.h +++ b/inc/ti/fn/fncommit.h @@ -3,8 +3,8 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) { const int nargs = fn_get_nargs(nd); - vec_t ** migs = ti_query_migs(query); - ti_mig_t * mig; + vec_t ** commits = ti_query_commits(query); + ti_commit_t * commit; if (fn_not_thingsdb_or_collection_scope("commit", query, e) || fn_nargs("commit", DOC_COMMIT, 1, nargs, e) || @@ -15,7 +15,7 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) raw = (ti_raw_t *) query->rval; query->rval = NULL; - if (!(*migs)) + if (!(*commits)) { ex_set(e, EX_OPERATION, "commit history is not enabled for the `%s` scope" @@ -24,24 +24,6 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) goto fail0; } - if (!query->change) - { - ex_set(e, EX_OPERATION, "commit without a change"DOC_SET_HISTORY); - goto fail0; - } - - if (!raw->n) - { - ex_set(e, EX_VALUE_ERROR, "commit message must not be empty"); - goto fail0; - } - - if (query->mig) - { - ex_set(e, EX_OPERATION, "commit message already set"); - goto fail0; - } - switch ((ti_query_with_enum) query->with_tp) { case TI_QUERY_WITH_PARSERES: break; @@ -60,12 +42,39 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) goto fail0; } - mig = ti_mig_create_q( - query->change->id, query->with.parseres->str, query->user->name); + if (!query->change) + { + ex_set(e, EX_OPERATION, "commit without a change"DOC_SET_HISTORY); + goto fail0; + } + + if (!query->user) + { + ex_set(e, EX_OPERATION, "commit without a user"DOC_SET_HISTORY); + goto fail0; + } + + if (!raw->n) + { + ex_set(e, EX_VALUE_ERROR, "commit message must not be empty"); + goto fail0; + } + + if (query->commit) + { + ex_set(e, EX_OPERATION, "commit message already set"); + goto fail0; + } + + commit = ti_commit_make( + query->change->id, + query->with.parseres->str, + query->user->name, + raw); - if (!mig || vec_push(migs, mig)) + if (!commit || vec_push(commits, commit)) { - ti_mig_destroy(mig); + ti_commit_destroy(commit); ex_set_mem(e); goto fail0; } diff --git a/inc/ti/fn/fnhistory.h b/inc/ti/fn/fnhistory.h new file mode 100644 index 00000000..aeb6142e --- /dev/null +++ b/inc/ti/fn/fnhistory.h @@ -0,0 +1,37 @@ +#include + + + + + +static int do__f_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) +{ + const int nargs = fn_get_nargs(nd); + vec_t ** commits; + vec_t * arr; + + if (fn_not_thingsdb_scope("history", query, e) || + ti_access_check_err( + ti.access_thingsdb, + query->user, TI_AUTH_GRANT, e) || + fn_nargs("history", DOC_HISTORY, 1, nargs, e) || + ti_do_statement(query, nd->children, e) || + fn_arg_thing("history", DOC_HISTORY, 1, query->rval, e)) + return e->nr; + + options.e = e; + + if (ti_thing_walk( + (ti_thing_t *) query->rval, + (ti_thing_item_cb) history__option, + &options)) + return e->nr; + + arr + + + + + + return e->nr; +} diff --git a/inc/ti/fn/fnsethistory.h b/inc/ti/fn/fnsethistory.h new file mode 100644 index 00000000..6fdae67d --- /dev/null +++ b/inc/ti/fn/fnsethistory.h @@ -0,0 +1,50 @@ +#include + +static int do__f_set_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) +{ + const int nargs = fn_get_nargs(nd); + _Bool state; + ti_task_t * task; + uint64_t mask, scope_id; + vec_t ** access_, ** commits; + + if (fn_not_thingsdb_scope("set_history", query, e) || + fn_nargs("set_history", DOC_SET_HISTORY, 2, nargs, e) || + ti_do_statement(query, nd->children, e)) + return e->nr; + + access_ = ti_val_get_access(query->rval, e, &scope_id); + if (e->nr || ti_access_check_err(*access_, query->user, TI_AUTH_GRANT, e)) + return e->nr; + + commits = ti_commits_from_scope((ti_raw_t *) query->rval, e); + if (e->nr) + return e->nr; + + ti_val_unsafe_drop(query->rval); + query->rval = NULL; + + /* read state */ + if (ti_do_statement(query, nd->children->next->next, e) || + fn_arg_bool("set_history", DOC_SET_HISTORY, 2, query->rval, e)) + return e->nr; + + state = VBOOL(query->rval); + ti_val_unsafe_drop(query->rval); + query->rval = (ti_val_t *) ti_nil_get(); + + if (ti_commits_set_history(commits, state)) + { + ex_set_mem(e); + return e->nr; + } + + task = ti_task_get_task( + query->change, + query->collection ? query->collection->root : ti.thing0); + + if (!task || ti_task_add_set_history(task, state)) + ex_set_mem(e); /* task cleanup is not required */ + + return e->nr; +} diff --git a/inc/ti/mig.h b/inc/ti/mig.h deleted file mode 100644 index d110e996..00000000 --- a/inc/ti/mig.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * ti/mig.h - */ -#ifndef TI_MIG_H_ -#define TI_MIG_H_ - -#include -#include -#include -#include -#include -#include - -typedef struct ti_mig_s ti_mig_t; - -ti_mig_t * ti_mig_create( - uint64_t id, - time_t ts, - const char * query, - size_t query_n, - const char * info, - size_t info_n, - const char * by, - size_t by_n, - const char * err_msg, /* may be NULL*/ - size_t err_msg_n); -ti_mig_t * ti_mig_create_q( - uint64_t id, - const char * query, - ti_raw_t * by); -void ti_mig_destroy(ti_mig_t * mig); -int ti_mig_set_history(vec_t ** migs, _Bool state); - -struct ti_mig_s -{ - uint64_t id; /* equal to the change id */ - time_t ts; /* timestamp of the change */ - ti_raw_t * query; /* original query string */ - ti_raw_t * info; /* never NULL */ - ti_raw_t * by; /* never NULL */ - ti_raw_t * err_msg; /* may be NULL */ -} - -#endif /* TI_MIG_H_ */ diff --git a/inc/ti/query.inline.h b/inc/ti/query.inline.h index a3087448..66ca43ca 100644 --- a/inc/ti/query.inline.h +++ b/inc/ti/query.inline.h @@ -72,11 +72,11 @@ static inline uint64_t ti_query_scope_id(ti_query_t * query) : TI_SCOPE_NODE; } -static inline vec_t ** ti_query_migs(ti_query_t * query) +static inline vec_t ** ti_query_commits(ti_query_t * query) { return query->collection - ? &query->collection->migs - : &ti.migs; + ? &query->collection->commits + : &ti.commits; } static inline int ti_query_test_vset_operation(ti_query_t * query, ex_t * e) diff --git a/inc/ti/query.t.h b/inc/ti/query.t.h index c5c8d79f..c5f56b44 100644 --- a/inc/ti/query.t.h +++ b/inc/ti/query.t.h @@ -14,7 +14,7 @@ typedef int (*ti_query_vars_walk_cb)(void * data, void * arg); #include #include #include -#include +#include #include #include #include @@ -92,7 +92,7 @@ struct ti_query_s ti_change_t * change; /* with reference, only when a change is required otherwise NULL */ - ti_mig_t * mig; /* NULL if no migration change */ + ti_commit_t * commit; /* NULL if no migration change */ vec_t * immutable_cache; /* ti_val_t, Only for immutable and collection independent variable and temporary used for procedures to populate the closure arguments diff --git a/inc/ti/task.h b/inc/ti/task.h index 47aaae05..f6ffcfab 100644 --- a/inc/ti/task.h +++ b/inc/ti/task.h @@ -188,6 +188,6 @@ int ti_task_add_whitelist_del( ti_user_t * user, ti_val_t * val, int wid); -int ti_task_add_mig_add(ti_task_t * task, ti_mig_t * mig); +int ti_task_add_commit_add(ti_task_t * task, ti_commit_t * commit); #endif /* TI_TASK_H_ */ diff --git a/inc/ti/task.t.h b/inc/ti/task.t.h index b2ed733a..f0ecfe0e 100644 --- a/inc/ti/task.t.h +++ b/inc/ti/task.t.h @@ -93,8 +93,8 @@ typedef enum TI_TASK_WHITELIST_ADD, /* 80 */ TI_TASK_WHITELIST_DEL, /* 81 */ TI_TASK_SET_HISTORY, /* 82 */ - TI_TASK_MIG_ADD, /* 83 */ - TI_TASK_MIG_DEL, /* 84 */ + TI_TASK_COMMIT_DEL, /* 83 */ + TI_TASK_COMMIT_ADD, /* 84 */ } ti_task_enum; typedef struct ti_task_s ti_task_t; diff --git a/src/ti.c b/src/ti.c index 3378f782..cda447b7 100644 --- a/src/ti.c +++ b/src/ti.c @@ -89,7 +89,7 @@ int ti_create(void) ti.node = NULL; ti.store = NULL; ti.tasks = NULL; - ti.migs = NULL; + ti.commits = NULL; ti.access_node = vec_new(0); ti.access_thingsdb = vec_new(0); ti.procedures = smap_create(); @@ -956,7 +956,7 @@ int ti_this_node_to_pk(msgpack_packer * pk) const char * architecture = osarch_get_arch(); return ( - msgpack_pack_map(pk, 41) || + msgpack_pack_map(pk, 42) || /* 1 */ mp_pack_str(pk, "node_id") || msgpack_pack_uint32(pk, ti.node->id) || @@ -1083,9 +1083,15 @@ int ti_this_node_to_pk(msgpack_packer * pk) /* 40 */ mp_pack_str(pk, "next_free_id") || msgpack_pack_uint64(pk, ti.node->next_free_id) || - /* 4 */ + /* 41 */ mp_pack_str(pk, "libwebsockets_version") || - mp_pack_str(pk, lws_get_library_version()) + mp_pack_str(pk, lws_get_library_version() || + /* 42 */ + mp_pack_str(pk, "commit_history") || + (ti.commits + ? msgpack_pack_uint32(pk, ti.commits->n) + : mp_pack_str(pk, "disabled")) + ) ); } diff --git a/src/ti/commit.c b/src/ti/commit.c new file mode 100644 index 00000000..0d6d1da5 --- /dev/null +++ b/src/ti/commit.c @@ -0,0 +1,110 @@ +/* + * ti/commit.c + */ +#include +#include +#include +#include + +ti_commit_t * ti_commit_create( + uint64_t id, + time_t ts, + const char * code, + size_t code_n, + const char * message, + size_t message_n, + const char * by, + size_t by_n, + const char * err_msg, + size_t err_msg_n) +{ + ti_commit_t * commit = malloc(sizeof(ti_commit_t)); + if (!commit) + return NULL; + + commit->id = id; + commit->ts = ts; + + commit->code = ti_str_create(code, code_n); + if (!commit->code) + goto fail0; + + commit->message = ti_str_create(message, message_n); + if (!commit->message) + goto fail1; + + commit->by = ti_str_create(by, by_n); + if (!commit->by) + goto fail2; + + if (err_msg) + { + commit->err_msg = ti_str_create(err_msg, err_msg_n); + if (!commit->err_msg) + goto fail3; + } + else + commit->err_msg = NULL; + + return commit; +fail3: + ti_val_unsafe_drop(commit->by); +fail2: + ti_val_unsafe_drop(commit->message); +fail1: + ti_val_unsafe_drop(commit->code); +fail0: + free(commit); +} + +ti_commit_t * ti_commit_make( + uint64_t id, + const char * code, + ti_raw_t * by, + ti_raw_t * message) +{ + ti_commit_t * commit = malloc(sizeof(ti_commit_t)); + if (!commit) + return NULL; + + commit->id = id; + commit->ts = (time_t) util_now_usec(); + + commit->code = ti_str_from_str(code); + if (!commit->code) + { + free(commit); + return NULL; + } + + commit->by = ti_grab(by); + commit->message = ti_grab(message); + commit->err_msg = NULL; + return commit; +} + +void ti_commit_destroy(ti_commit_t * commit) +{ + if (!commit) + return; + ti_val_unsafe_drop(commit->code); + ti_val_unsafe_drop(commit->message); + ti_val_unsafe_drop(commit->by); + ti_val_drop(commit->err_msg); + free(commit); +} + +int ti_commit_set_history(vec_t ** commits, _Bool state) +{ + if (state) + { + if (*commits) + return 0; + + *commits = vec_new(8); + return !!(*commits); + } + vec_destroy(*commits, ti_commit_destroy); + *commits = NULL; + return 0; +} \ No newline at end of file diff --git a/src/ti/commits.c b/src/ti/commits.c new file mode 100644 index 00000000..822ccacb --- /dev/null +++ b/src/ti/commits.c @@ -0,0 +1,59 @@ +/* + * ti/commits.c + */ +#include +#include + +typedef struct +{ + ti_raw_t * scope; + ti_raw_t * contains; + ti_regex_t * match; + ti_vint_t * id; + ti_vint_t * last; + ti_vint_t * first; + ti_vbool_t * detail; + ti_datetime_t * before; + ti_datetime_t * after; + ex_t * e; +} commits__options_t; + +vec_t ** ti_commits_from_scope(ti_raw_t * scope, ex_t * e) +{ + if (ti_scope_init(&scope, (const char *) scope->data, scope->n, e)) + return NULL; + + switch(scope.tp) + { + case TI_SCOPE_THINGSDB: + return &ti.commits; + case TI_SCOPE_NODE: + return NULL; + case TI_SCOPE_COLLECTION: + collection = ti_collections_get_by_strn( + scope.via.collection_name.name, + scope.via.collection_name.sz); + if (collection) + return &collection->access; + + ex_set(e, EX_LOOKUP_ERROR, "collection `%.*s` not found", + scope.via.collection_name.sz, + scope.via.collection_name.name); + } + return NULL; +} + +int ti_commits_set_history(vec_t ** commits, _Bool state) +{ + if (state) + { + if (*commits) + return 0; + + *commits = vec_new(8); + return !!(*commits); + } + vec_destroy(*commits, ti_commit_destroy); + *commits = NULL; + return 0; +} \ No newline at end of file diff --git a/src/ti/ctask.c b/src/ti/ctask.c index 2a025b68..1e51edcc 100644 --- a/src/ti/ctask.c +++ b/src/ti/ctask.c @@ -3280,6 +3280,70 @@ int ctask__arr_remove(ti_thing_t * thing, mp_unp_t * up) return 0; } +/* + * Returns 0 on success + * - for example: '{name: closure}' + */ +static int ctask__commit_add(ti_thing_t * thing, mp_unp_t * up) +{ + int rc; + mp_obj_t obj, mp_name, mp_created; + ti_collection_t * collection = thing->collection; + ti_procedure_t * procedure; + ti_closure_t * closure; + ti_vup_t vup = { + .isclient = false, + .collection = collection, + .up = up, + }; + + if (mp_next(up, &obj) != MP_MAP || obj.via.sz != 3 || + mp_skip(up) != MP_STR || + mp_next(up, &mp_name) != MP_STR || + mp_skip(up) != MP_STR || + mp_next(up, &mp_created) != MP_U64 || + mp_skip(up) != MP_STR) + { + log_critical( + "task `new_procedure` for "TI_COLLECTION_ID": " + "missing map or name", + collection->id); + return -1; + } + + closure = (ti_closure_t *) ti_val_from_vup(&vup); + procedure = NULL; + + if (!closure || !ti_val_is_closure((ti_val_t *) closure) || + !(procedure = ti_procedure_create( + mp_name.via.str.data, + mp_name.via.str.n, + closure, + mp_created.via.u64))) + goto failed; + + rc = ti_procedures_add(collection->procedures, procedure); + if (rc == 0) + { + ti_decref(closure); + return 0; /* success */ + } + + if (rc < 0) + log_critical(EX_MEMORY_S); + else + log_critical( + "task `new_procedure` for "TI_COLLECTION_ID": " + "procedure `%s` already exists", + collection->id, + procedure->name->str); + +failed: + ti_procedure_destroy(procedure); + ti_val_drop((ti_val_t *) closure); + return -1; +} + /* * Unpacker should be at point 'task': ... */ @@ -3384,6 +3448,9 @@ int ti_ctask_run(ti_thing_t * thing, mp_unp_t * up) case TI_TASK_ROOM_SET_NAME: return ctask__room_set_name(thing, up); case TI_TASK_WHITELIST_ADD: break; case TI_TASK_WHITELIST_DEL: break; + case TI_TASK_SET_HISTORY: break; + case TI_TASK_COMMIT_DEL: break; + case TI_TASK_COMMIT_ADD: return ctask__commit_add(thing, up); } log_critical("unknown collection task: %"PRIu64, mp_task.via.u64); diff --git a/src/ti/export.c b/src/ti/export.c index ba3e4c8e..2d191b0c 100644 --- a/src/ti/export.c +++ b/src/ti/export.c @@ -646,9 +646,19 @@ static int export__write_to_type(ti_fmt_t * fmt, ti_collection_t * collection) : 0; } +static int export__commit(ti_fmt_t * fmt, ti_collection_t * collection) +{ + return buf_append_fmt( + &fmt->buf, + "try(commit('Source: collection `%.*s`'));\n\n", + collection->name->n, collection->name-data); +} + static int export__collection(ti_fmt_t * fmt, ti_collection_t * collection) { + return -( + export__commit(fmt, collection) || export__write_new_types(fmt, collection->types) || export__write_enums(fmt, collection->enums) || export__write_types(fmt, collection->types) || diff --git a/src/ti/mig.c b/src/ti/mig.c deleted file mode 100644 index ccef1717..00000000 --- a/src/ti/mig.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * ti/mig.c - */ -#include -#include -#include -#include - -ti_mig_t * ti_mig_create( - uint64_t id, - time_t ts, - const char * query, - size_t query_n, - const char * info, - size_t info_n, - const char * by, - size_t by_n, - const char * err_msg, - size_t err_msg_n) -{ - ti_mig_t * mig = malloc(sizeof(ti_mig_t)); - if (!mig) - return NULL; - - mig->id = id; - mig->ts = ts; - - mig->query = ti_str_create(query, query_n); - if (!mig->query) - goto fail0; - - mig->info = ti_str_create(info, info_n); - if (!mig->info) - goto fail1; - - mig->by = ti_str_create(by, by_n); - if (!mig->by) - goto fail2; - - if (err_msg) - { - mig->err_msg = ti_str_create(err_msg, err_msg_n); - if (!mig->err_msg) - goto fail3; - } - else - mig->err_msg = NULL; - - return mig; -fail3: - ti_val_unsafe_drop(mig->by); -fail2: - ti_val_unsafe_drop(mig->info); -fail1: - ti_val_unsafe_drop(mig->query); -fail0: - free(mig); -} - -ti_mig_t * ti_mig_create_q( - uint64_t id, - const char * query, - ti_raw_t * by) -{ - ti_mig_t * mig = malloc(sizeof(ti_mig_t)); - if (!mig) - return NULL; - - mig->id = id; - mig->ts = (time_t) util_now_usec(); - - mig->query = ti_str_from_str(query); - if (!mig->query) - { - free(mig); - return NULL; - } - - mig->info = ti_val_empty_str(); - mig->by = ti_grab(by); - mig->err_msg = NULL; - return mig; -} - -void ti_mig_destroy(ti_mig_t * mig) -{ - if (!mig) - return; - ti_val_unsafe_drop(mig->query); - ti_val_unsafe_drop(mig->info); - ti_val_unsafe_drop(mig->by); - ti_val_drop(mig->err_msg); - free(mig); -} - -int ti_mig_set_history(vec_t ** migs, _Bool state) -{ - if (state) - { - if (*migs) - return 0; - - *migs = vec_new(8); - return !!(*migs); - } - vec_destroy(*migs, ti_mig_destroy); - *migs = NULL; - return 0; -} \ No newline at end of file diff --git a/src/ti/query.c b/src/ti/query.c index 99deb19c..ea8bc781 100644 --- a/src/ti/query.c +++ b/src/ti/query.c @@ -955,19 +955,19 @@ void ti_query_run_parseres(ti_query_t * query) stop: if (query->change) { - if (query->mig) + if (query->commit) { ti_task_t * task; if (e.nr) /* set error if any, not critical if failed */ - query->mig->err_msg = ti_str_create(e.msg, e.n); + query->commit->err_msg = ti_str_create(e.msg, e.n); task = ti_task_get_task( query->change, query->collection ? query->collection->root : ti.thing0); if (task) - (void) ti_task_add_mig_add(task, query->mig); + (void) ti_task_add_commit_add(task, query->commit); } query__change_handle(query); /* errors will be logged only */ } diff --git a/src/ti/task.c b/src/ti/task.c index 1c32a651..dae617a6 100644 --- a/src/ti/task.c +++ b/src/ti/task.c @@ -3140,10 +3140,10 @@ int ti_task_add_whitelist_del( return -1; } -int ti_task_add_mig_add(ti_task_t * task, ti_mig_t * mig) +int ti_task_add_commit_add(ti_task_t * task, ti_commit_t * commit) { - size_t alloc = (50 + mig->query->n + mig->info->n + mig->by->n + ( - mig->err_msg ? mig->err_msg->n : 0) + size_t alloc = (50 + commit->code->n + commit->info->n + commit->by->n + ( + commit->err_msg ? commit->err_msg->n : 0) ); ti_data_t * data; msgpack_packer pk; @@ -3155,16 +3155,16 @@ int ti_task_add_mig_add(ti_task_t * task, ti_mig_t * mig) msgpack_pack_array(pk, 2); - msgpack_pack_uint8(pk, TI_TASK_MIG_ADD); - msgpack_pack_array(pk, 5 + !!mig->err_msg); + msgpack_pack_uint8(pk, TI_TASK_COMMIT_ADD); + msgpack_pack_array(pk, 5 + !!commit->err_msg); - msgpack_pack_uint64(pk, mig->id); - msgpack_pack_uint64(pk, (uint64_t) mig->ts); - mp_pack_strn(pk, mig->query->data, mig->query->n); - mp_pack_strn(pk, mig->info->data, mig->info->n); - mp_pack_strn(pk, mig->by->data, mig->by->n); - if (mig->err_msg) - mp_pack_strn(pk, mig->err_msg->data, mig->err_msg->n); + msgpack_pack_uint64(pk, commit->id); + msgpack_pack_uint64(pk, (uint64_t) commit->ts); + mp_pack_strn(pk, commit->code->data, commit->code->n); + mp_pack_strn(pk, commit->message->data, commit->message->n); + mp_pack_strn(pk, commit->by->data, commit->by->n); + if (commit->err_msg) + mp_pack_strn(pk, commit->err_msg->data, commit->err_msg->n); if (vec_push(&task->list, data)) goto fail_data; diff --git a/src/ti/ttask.c b/src/ti/ttask.c index b42bf934..e43a4199 100644 --- a/src/ti/ttask.c +++ b/src/ti/ttask.c @@ -705,7 +705,7 @@ static int ttask__whitelist_del(mp_unp_t * up) static int ttask__set_historyl(mp_unp_t * up) { - vec_t ** migs; + vec_t ** commits; mp_obj_t obj, mp_scope, mp_state; if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 2 @@ -728,14 +728,14 @@ static int ttask__set_historyl(mp_unp_t * up) mp_scope.via.u64); return -1; } - migs = &collection->migs; + commits = &collection->commits; } else { - migs = &ti.migs; + commits = &ti.commits; } - if (ti_mig_set_history(&migs, mp_state.via.bool_)) + if (ti_commit_set_history(&commits, mp_state.via.bool_)) { log_critical("failed to set `set_history`"); return -1; @@ -1777,8 +1777,8 @@ int ti_ttask_run(ti_change_t * change, mp_unp_t * up) case TI_TASK_WHITELIST_ADD: return ttask__whitelist_add(up); case TI_TASK_WHITELIST_DEL: return ttask__whitelist_del(up); case TI_TASK_SET_HISTORY: return ttask__set_history(up); - case TI_TASK_MIG_ADD: return ttask__mig_add(up); - case TI_TASK_MIG_DEL: return ttask__mig_del(up); + case TI_TASK_COMMIT_DEL: return ttask__commit_del(up); + case TI_TASK_COMMIT_ADD: return ttask__commit_add(up); } log_critical("unknown thingsdb task: %"PRIu64, mp_task.via.u64); From 25b8374f4334fd38a710e8edabf3a64a3f498d86 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Tue, 4 Nov 2025 17:10:00 +0100 Subject: [PATCH 07/23] work on mig --- CMakeLists.txt | 5 +- inc/ti/commit.h | 21 +- inc/ti/commits.h | 35 +++- inc/ti/fn/fn.h | 3 +- inc/ti/fn/fncommit.h | 23 ++- inc/ti/fn/fndelhistory.h | 46 +++++ inc/ti/fn/fndelprocedure.h | 1 + inc/ti/fn/fndeltype.h | 1 + inc/ti/fn/fnhistory.h | 47 +++-- inc/ti/fn/fnmodenum.h | 1 + inc/ti/fn/fnmodprocedure.h | 1 + inc/ti/fn/fnmodtype.h | 1 + inc/ti/fn/fnnewprocedure.h | 1 + inc/ti/fn/fnrenameenum.h | 1 + inc/ti/fn/fnrenameprocedure.h | 1 + inc/ti/fn/fnrenametype.h | 1 + inc/ti/fn/fnsetenum.h | 1 + inc/ti/fn/fnsethistory.h | 4 +- inc/ti/fn/fntotype.h | 12 ++ inc/ti/store.h | 1 + inc/ti/store/storecollection.h | 5 +- inc/ti/store/storecommits.h | 13 ++ inc/ti/task.h | 5 + inc/ti/task.t.h | 2 +- src/ti.c | 6 +- src/ti/collection.c | 1 + src/ti/commit.c | 118 +++++++++-- src/ti/commits.c | 346 +++++++++++++++++++++++++++++++-- src/ti/ctask.c | 72 +++---- src/ti/export.c | 2 +- src/ti/qbind.c | 70 ++++--- src/ti/store.c | 22 ++- src/ti/store/storecollection.c | 26 ++- src/ti/store/storecommits.c | 128 ++++++++++++ src/ti/syncfull.c | 6 + src/ti/task.c | 95 +++++++-- src/ti/ttask.c | 93 ++++++++- 37 files changed, 1024 insertions(+), 193 deletions(-) create mode 100644 inc/ti/fn/fndelhistory.h create mode 100644 inc/ti/store/storecommits.h create mode 100644 src/ti/store/storecommits.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 87d60ee6..2923ff8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,6 +137,8 @@ set(SOURCES src/ti/closure.c src/ti/collection.c src/ti/collections.c + src/ti/commit.c + src/ti/commits.c src/ti/condition.c src/ti/connect.c src/ti/counters.c @@ -238,11 +240,12 @@ set(SOURCES src/ti/store/storeaccess.c src/ti/store/storecollection.c src/ti/store/storecollections.c + src/ti/store/storecommits.c src/ti/store/storeenums.c src/ti/store/storegcollect.c src/ti/store/storemodules.c - src/ti/store/storenames.c src/ti/store/storenamedrooms.c + src/ti/store/storenames.c src/ti/store/storeprocedures.c src/ti/store/storestatus.c src/ti/store/storetasks.c diff --git a/inc/ti/commit.h b/inc/ti/commit.h index 0308cddc..0c6dde6f 100644 --- a/inc/ti/commit.h +++ b/inc/ti/commit.h @@ -8,28 +8,25 @@ #include #include #include +#include #include #include typedef struct ti_commit_s ti_commit_t; -ti_commit_t * ti_commit_create( - uint64_t id, - time_t ts, - const char * code, - size_t code_n, - const char * message, - size_t message_n, - const char * by, - size_t by_n, - const char * err_msg, /* may be NULL*/ - size_t err_msg_n); +ti_commit_t * ti_commit_from_up(mp_unp_t * up); ti_commit_t * ti_commit_make( uint64_t id, const char * code, ti_raw_t * by, ti_raw_t * message); void ti_commit_destroy(ti_commit_t * commit); +int ti_commit_to_pk(ti_commit_t * commit, msgpack_packer * pk); +int ti_commit_to_client_pk( + ti_commit_t * commit, + _Bool detail, + msgpack_packer * pk); +ti_val_t * ti_commit_as_mpval(ti_commit_t * commit, _Bool detail); struct ti_commit_s { @@ -39,6 +36,6 @@ struct ti_commit_s ti_raw_t * message; /* never NULL */ ti_raw_t * by; /* never NULL */ ti_raw_t * err_msg; /* may be NULL */ -} +}; #endif /* TI_COMMIT_H_ */ diff --git a/inc/ti/commits.h b/inc/ti/commits.h index 16661a1d..b85b355a 100644 --- a/inc/ti/commits.h +++ b/inc/ti/commits.h @@ -14,8 +14,39 @@ #include #include +typedef struct +{ + ti_raw_t * scope; + ti_raw_t * contains; + ti_regex_t * match; + ti_vint_t * id; + ti_vint_t * last; + ti_vint_t * first; + ti_datetime_t * before; + ti_datetime_t * after; + ti_vbool_t * detail; +} ti_commits_options_t; + +typedef struct +{ + vec_t ** commits; + vec_t ** access; + uint64_t scope_id; +} ti_commits_history_t; + +void ti_commits_destroy(vec_t ** commits); +int ti_commits_options( + ti_commits_options_t * options, + ti_thing_t * thing, + _Bool allow_detail, + ex_t * e); +int ti_commits_history( + ti_commits_history_t * history, + ti_commits_options_t * options, + ex_t * e); +vec_t ** ti_commits_from_scope(ti_raw_t * scope, ex_t * e); int ti_commits_set_history(vec_t ** commits, _Bool state); -ti_varr_t * ti_commits_find(vec_t ** commits, ti_thing_t * thing); -vec_t * ti_commits_del(vec_t ** commits, ti_thing_t * thing); +vec_t * ti_commits_find(vec_t * commits, ti_commits_options_t * options); +vec_t * ti_commits_del(vec_t ** commits, ti_commits_options_t * options); #endif /* TI_COMMITS_H_ */ diff --git a/inc/ti/fn/fn.h b/inc/ti/fn/fn.h index 424878ca..65b839fa 100644 --- a/inc/ti/fn/fn.h +++ b/inc/ti/fn/fn.h @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -444,7 +445,7 @@ static inline int fn_arg_name_check( return e->nr; } -static int fn_commit(const char * name, ti_query_t * query, ex_t * e) +static inline int fn_commit(const char * name, ti_query_t * query, ex_t * e) { if (ti_query_commits(query) && !query->commit) ex_set(e, EX_OPERATION, diff --git a/inc/ti/fn/fncommit.h b/inc/ti/fn/fncommit.h index 73748d85..a8155261 100644 --- a/inc/ti/fn/fncommit.h +++ b/inc/ti/fn/fncommit.h @@ -3,6 +3,8 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) { const int nargs = fn_get_nargs(nd); + ti_task_t * task; + ti_raw_t * message; vec_t ** commits = ti_query_commits(query); ti_commit_t * commit; @@ -12,8 +14,8 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) fn_arg_str("commit", DOC_COMMIT, 1, query->rval, e)) return e->nr; - raw = (ti_raw_t *) query->rval; - query->rval = NULL; + message = (ti_raw_t *) query->rval; + query->rval = (ti_val_t *) ti_nil_get(); if (!(*commits)) { @@ -54,7 +56,7 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) goto fail0; } - if (!raw->n) + if (!message->n) { ex_set(e, EX_VALUE_ERROR, "commit message must not be empty"); goto fail0; @@ -70,7 +72,7 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) query->change->id, query->with.parseres->str, query->user->name, - raw); + message); if (!commit || vec_push(commits, commit)) { @@ -79,8 +81,17 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) goto fail0; } - query->rval = ti_nil_get(); + task = ti_task_get_task( + query->change, + query->collection ? query->collection->root : ti.thing0); + + if (!task || ti_task_add_commit_add(task, commit)) + { + ti_commit_destroy(vec_pop(*commits)); /* undo commit */ + ex_set_mem(e); + } + fail0: - ti_val_unsafe_drop((ti_val_t *) raw); + ti_val_unsafe_drop((ti_val_t *) message); return e->nr; } diff --git a/inc/ti/fn/fndelhistory.h b/inc/ti/fn/fndelhistory.h new file mode 100644 index 00000000..e3e4bb87 --- /dev/null +++ b/inc/ti/fn/fndelhistory.h @@ -0,0 +1,46 @@ +#include + +static int do__f_del_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) +{ + const int nargs = fn_get_nargs(nd); + ti_commits_options_t options = {0}; + ti_commits_history_t history = {0}; + vec_t * deleted; + ti_task_t * task; + + if (fn_not_thingsdb_scope("del_history", query, e) || + fn_nargs("del_history", DOC_DEL_HISTORY, 1, nargs, e) || + ti_do_statement(query, nd->children, e) || + fn_arg_thing("del_history", DOC_DEL_HISTORY, 1, query->rval, e) || + ti_commits_options(&options, (ti_thing_t *) query->rval, false, e) || + ti_commits_history(&history, &options, e)) + return e->nr; + + if (ti_access_check_err(*history.access, query->user, TI_AUTH_GRANT, e)) + return e->nr; + + deleted = ti_commits_del(history.commits, &options); + if (!deleted) + goto fail; + + task = ti_task_get_task( + query->change, + query->collection ? query->collection->root : ti.thing0); + if (!task || ti_task_add_del_history(task, history.scope_id, deleted)) + goto undo; + + ti_val_unsafe_drop(query->rval); + query->rval = (ti_val_t *) ti_vint_create((int64_t) deleted->n); + if (!query->rval) + ex_set_mem(e); + + vec_destroy(deleted, (vec_destroy_cb) ti_commit_destroy); + return e->nr; +undo: + for (vec_each(deleted, ti_commit_t, commit)) + VEC_push(*history.commits, commit); + vec_destroy(deleted, NULL); +fail: + ex_set_mem(e); + return e->nr; +} diff --git a/inc/ti/fn/fndelprocedure.h b/inc/ti/fn/fndelprocedure.h index 9273dbb1..89ab6920 100644 --- a/inc/ti/fn/fndelprocedure.h +++ b/inc/ti/fn/fndelprocedure.h @@ -8,6 +8,7 @@ static int do__f_del_procedure(ti_query_t * query, cleri_node_t * nd, ex_t * e) smap_t * procedures = ti_query_procedures(query); if (fn_not_thingsdb_or_collection_scope("del_procedure", query, e) || + fn_commit("del_procedure", query, e) || fn_nargs("del_procedure", DOC_DEL_PROCEDURE, 1, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("del_procedure", DOC_DEL_PROCEDURE, 1, query->rval, e)) diff --git a/inc/ti/fn/fndeltype.h b/inc/ti/fn/fndeltype.h index c66c0258..67678a53 100644 --- a/inc/ti/fn/fndeltype.h +++ b/inc/ti/fn/fndeltype.h @@ -7,6 +7,7 @@ static int do__f_del_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) ti_task_t * task; if (fn_not_collection_scope("del_type", query, e) || + fn_commit("del_type", query, e) || fn_nargs("del_type", DOC_DEL_TYPE, 1, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("del_type", DOC_DEL_TYPE, 1, query->rval, e)) diff --git a/inc/ti/fn/fnhistory.h b/inc/ti/fn/fnhistory.h index aeb6142e..6c98e725 100644 --- a/inc/ti/fn/fnhistory.h +++ b/inc/ti/fn/fnhistory.h @@ -1,37 +1,48 @@ #include - - - - static int do__f_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) { const int nargs = fn_get_nargs(nd); - vec_t ** commits; - vec_t * arr; + ti_commits_options_t options = {0}; + ti_commits_history_t history = {0}; + vec_t * filtered; + _Bool detail; + uint32_t i = 0; if (fn_not_thingsdb_scope("history", query, e) || - ti_access_check_err( - ti.access_thingsdb, - query->user, TI_AUTH_GRANT, e) || fn_nargs("history", DOC_HISTORY, 1, nargs, e) || ti_do_statement(query, nd->children, e) || - fn_arg_thing("history", DOC_HISTORY, 1, query->rval, e)) + fn_arg_thing("history", DOC_HISTORY, 1, query->rval, e) || + ti_commits_options(&options, (ti_thing_t *) query->rval, true, e) || + ti_commits_history(&history, &options, e)) return e->nr; - options.e = e; - - if (ti_thing_walk( - (ti_thing_t *) query->rval, - (ti_thing_item_cb) history__option, - &options)) + if (ti_access_check_err(*history.access, query->user, TI_AUTH_GRANT, e)) return e->nr; - arr - + filtered = ti_commits_find(*history.commits, &options); + if (!filtered) + goto fail; + detail = options.detail && options.detail->bool_; + for (vec_each_addr(filtered, void, commit), i++) + { + ti_val_t * v = ti_commit_as_mpval(*commit, detail); + if (!v) + goto fail; + *commit = v; + } + ti_val_unsafe_drop(query->rval); + query->rval = (ti_val_t *) ti_varr_from_vec(filtered); + if (!query->rval) + goto fail; return e->nr; +fail: + filtered->n = i; + vec_destroy(filtered, (vec_destroy_cb) ti_val_unassign_drop); + ex_set_mem(e); + return e->nr; } diff --git a/inc/ti/fn/fnmodenum.h b/inc/ti/fn/fnmodenum.h index 75235759..3ad01fdf 100644 --- a/inc/ti/fn/fnmodenum.h +++ b/inc/ti/fn/fnmodenum.h @@ -313,6 +313,7 @@ static int do__f_mod_enum(ti_query_t * query, cleri_node_t * nd, ex_t * e) const int nargs = fn_get_nargs(nd); if (fn_not_collection_scope("mod_enum", query, e) || + fn_commit("mod_enum", query, e) || fn_nargs_min("mod_enum", DOC_MOD_ENUM, 3, nargs, e) || ti_do_statement(query, child, e) || fn_arg_str_slow("mod_enum", DOC_MOD_ENUM, 1, query->rval, e)) diff --git a/inc/ti/fn/fnmodprocedure.h b/inc/ti/fn/fnmodprocedure.h index 08600c75..bfe15a0a 100644 --- a/inc/ti/fn/fnmodprocedure.h +++ b/inc/ti/fn/fnmodprocedure.h @@ -9,6 +9,7 @@ static int do__f_mod_procedure(ti_query_t * query, cleri_node_t * nd, ex_t * e) smap_t * procedures = ti_query_procedures(query); if (fn_not_thingsdb_or_collection_scope("mod_procedure", query, e) || + fn_commit("mod_procedure", query, e) || fn_nargs("mod_procedure", DOC_MOD_PROCEDURE, 2, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("mod_procedure", DOC_MOD_PROCEDURE, 1, query->rval, e)) diff --git a/inc/ti/fn/fnmodtype.h b/inc/ti/fn/fnmodtype.h index 4887dd6d..a6ddb551 100644 --- a/inc/ti/fn/fnmodtype.h +++ b/inc/ti/fn/fnmodtype.h @@ -1489,6 +1489,7 @@ static int do__f_mod_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) const int nargs = fn_get_nargs(nd); if (fn_not_collection_scope("mod_type", query, e) || + fn_commit("mod_type", query, e) || fn_nargs_min("mod_type", DOC_MOD_TYPE, 3, nargs, e) || ti_do_statement(query, child, e) || fn_arg_str_slow("mod_type", DOC_MOD_TYPE, 1, query->rval, e)) diff --git a/inc/ti/fn/fnnewprocedure.h b/inc/ti/fn/fnnewprocedure.h index a1336366..17dd78df 100644 --- a/inc/ti/fn/fnnewprocedure.h +++ b/inc/ti/fn/fnnewprocedure.h @@ -11,6 +11,7 @@ static int do__f_new_procedure(ti_query_t * query, cleri_node_t * nd, ex_t * e) smap_t * procedures = ti_query_procedures(query); if (fn_not_thingsdb_or_collection_scope("new_procedure", query, e) || + fn_commit("new_procedure", query, e) || fn_nargs("new_procedure", DOC_NEW_PROCEDURE, 2, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("new_procedure", DOC_NEW_PROCEDURE, 1, query->rval, e)) diff --git a/inc/ti/fn/fnrenameenum.h b/inc/ti/fn/fnrenameenum.h index 0a829a6f..b47b4d93 100644 --- a/inc/ti/fn/fnrenameenum.h +++ b/inc/ti/fn/fnrenameenum.h @@ -8,6 +8,7 @@ static int do__f_rename_enum(ti_query_t * query, cleri_node_t * nd, ex_t * e) ti_raw_t * oname, * nname; if (fn_not_collection_scope("rename_enum", query, e) || + fn_commit("rename_enum", query, e) || fn_nargs("rename_enum", DOC_RENAME_ENUM, 2, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("rename_enum", DOC_RENAME_ENUM, 1, query->rval, e)) diff --git a/inc/ti/fn/fnrenameprocedure.h b/inc/ti/fn/fnrenameprocedure.h index d0e815b7..c78dae6f 100644 --- a/inc/ti/fn/fnrenameprocedure.h +++ b/inc/ti/fn/fnrenameprocedure.h @@ -9,6 +9,7 @@ static int do__f_rename_procedure(ti_query_t * query, cleri_node_t * nd, ex_t * smap_t * procedures = ti_query_procedures(query); if (fn_not_thingsdb_or_collection_scope("rename_procedure", query, e) || + fn_commit("rename_procedure", query, e) || fn_nargs("rename_procedure", DOC_RENAME_PROCEDURE, 2, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("rename_procedure", DOC_RENAME_PROCEDURE, 1, query->rval, e)) diff --git a/inc/ti/fn/fnrenametype.h b/inc/ti/fn/fnrenametype.h index 5ff5823e..f40d262c 100644 --- a/inc/ti/fn/fnrenametype.h +++ b/inc/ti/fn/fnrenametype.h @@ -9,6 +9,7 @@ static int do__f_rename_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) if (fn_not_collection_scope("rename_type", query, e) || + fn_commit("rename_type", query, e) || fn_nargs("rename_type", DOC_RENAME_TYPE, 2, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("rename_type", DOC_RENAME_TYPE, 1, query->rval, e)) diff --git a/inc/ti/fn/fnsetenum.h b/inc/ti/fn/fnsetenum.h index 0f55e69f..7a105245 100644 --- a/inc/ti/fn/fnsetenum.h +++ b/inc/ti/fn/fnsetenum.h @@ -11,6 +11,7 @@ static int do__f_set_enum(ti_query_t * query, cleri_node_t * nd, ex_t * e) uint64_t ts_now = util_now_usec(); if (fn_not_collection_scope("set_enum", query, e) || + fn_commit("set_enum", query, e) || fn_nargs("set_enum", DOC_SET_ENUM, 2, nargs, e) || ti_do_statement(query, nd->children, e) || fn_arg_str("set_enum", DOC_SET_ENUM, 1, query->rval, e)) diff --git a/inc/ti/fn/fnsethistory.h b/inc/ti/fn/fnsethistory.h index 6fdae67d..8ff1564d 100644 --- a/inc/ti/fn/fnsethistory.h +++ b/inc/ti/fn/fnsethistory.h @@ -5,7 +5,7 @@ static int do__f_set_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) const int nargs = fn_get_nargs(nd); _Bool state; ti_task_t * task; - uint64_t mask, scope_id; + uint64_t scope_id; vec_t ** access_, ** commits; if (fn_not_thingsdb_scope("set_history", query, e) || @@ -43,7 +43,7 @@ static int do__f_set_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) query->change, query->collection ? query->collection->root : ti.thing0); - if (!task || ti_task_add_set_history(task, state)) + if (!task || ti_task_add_set_history(task, scope_id, state)) ex_set_mem(e); /* task cleanup is not required */ return e->nr; diff --git a/inc/ti/fn/fntotype.h b/inc/ti/fn/fntotype.h index 2d21930c..fd8b9f58 100644 --- a/inc/ti/fn/fntotype.h +++ b/inc/ti/fn/fntotype.h @@ -11,6 +11,18 @@ static int do__f_to_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) if (!ti_val_is_object(query->rval)) return fn_call_try("to_type", query, nd, e); + if (query->collection && + query->collection->root == (ti_thing_t *) query->rval && + query->collection->commits && + !query->commit) + { + ex_set(e, EX_OPERATION, + "function `to_type` requires a commit before " + "it can be used on the `root()` of the `%s` scope"DOC_COMMIT, + ti_query_scope_name(query)); + return e->nr; + } + if (fn_nargs("to_type", DOC_THING_TO_TYPE, 1, nargs, e) || ti_query_test_thing_operation(query, e)) return e->nr; diff --git a/inc/ti/store.h b/inc/ti/store.h index 541ae33a..0fa398b6 100644 --- a/inc/ti/store.h +++ b/inc/ti/store.h @@ -20,6 +20,7 @@ struct ti_store_s char * access_node_fn; char * access_thingsdb_fn; char * collections_fn; + char * commits_fn; char * id_stat_fn; char * names_fn; char * prev_path; diff --git a/inc/ti/store/storecollection.h b/inc/ti/store/storecollection.h index 8ec7b4b8..298e3296 100644 --- a/inc/ti/store/storecollection.h +++ b/inc/ti/store/storecollection.h @@ -49,7 +49,9 @@ char * ti_store_collection_gcthings_fn( char * ti_store_collection_named_rooms_fn( const char * path, uint64_t collection_id); - +char * ti_store_collection_commits_fn( + const char * path, + uint64_t collection_id); struct ti_store_collection_s { @@ -57,6 +59,7 @@ struct ti_store_collection_s char * collection_fn; char * props_fn; char * collection_path; + char * commits_fn; char * names_fn; char * named_rooms_fn; char * procedures_fn; diff --git a/inc/ti/store/storecommits.h b/inc/ti/store/storecommits.h new file mode 100644 index 00000000..e778e38f --- /dev/null +++ b/inc/ti/store/storecommits.h @@ -0,0 +1,13 @@ +/* + * ti/store/storecommits.h + */ +#ifndef TI_STORE_COMMITS_H_ +#define TI_STORE_COMMITS_H_ + +#include +#include + +int ti_store_commits_store(vec_t * commits, const char * fn); +int ti_store_commits_restore(vec_t ** commits, const char * fn); + +#endif /* TI_STORE_COMMITS_H_ */ diff --git a/inc/ti/task.h b/inc/ti/task.h index f6ffcfab..76a7f80f 100644 --- a/inc/ti/task.h +++ b/inc/ti/task.h @@ -189,5 +189,10 @@ int ti_task_add_whitelist_del( ti_val_t * val, int wid); int ti_task_add_commit_add(ti_task_t * task, ti_commit_t * commit); +int ti_task_add_set_history(ti_task_t * task, uint64_t scope_id, _Bool state); +int ti_task_add_del_history( + ti_task_t * task, + uint64_t scope_id, + vec_t * commits); #endif /* TI_TASK_H_ */ diff --git a/inc/ti/task.t.h b/inc/ti/task.t.h index f0ecfe0e..aa1103db 100644 --- a/inc/ti/task.t.h +++ b/inc/ti/task.t.h @@ -93,7 +93,7 @@ typedef enum TI_TASK_WHITELIST_ADD, /* 80 */ TI_TASK_WHITELIST_DEL, /* 81 */ TI_TASK_SET_HISTORY, /* 82 */ - TI_TASK_COMMIT_DEL, /* 83 */ + TI_TASK_DEL_HISTORY, /* 83 */ TI_TASK_COMMIT_ADD, /* 84 */ } ti_task_enum; diff --git a/src/ti.c b/src/ti.c index cda447b7..4a73d092 100644 --- a/src/ti.c +++ b/src/ti.c @@ -1085,13 +1085,13 @@ int ti_this_node_to_pk(msgpack_packer * pk) msgpack_pack_uint64(pk, ti.node->next_free_id) || /* 41 */ mp_pack_str(pk, "libwebsockets_version") || - mp_pack_str(pk, lws_get_library_version() || + mp_pack_str(pk, lws_get_library_version()) || /* 42 */ mp_pack_str(pk, "commit_history") || (ti.commits ? msgpack_pack_uint32(pk, ti.commits->n) - : mp_pack_str(pk, "disabled")) - ) + : mp_pack_str(pk, "disabled") + ) ); } diff --git a/src/ti/collection.c b/src/ti/collection.c index 0dd19b04..a0c22bad 100644 --- a/src/ti/collection.c +++ b/src/ti/collection.c @@ -100,6 +100,7 @@ void ti_collection_destroy(ti_collection_t * collection) ti_val_drop((ti_val_t *) collection->name); vec_destroy(collection->access, (vec_destroy_cb) ti_auth_destroy); vec_destroy(collection->vtasks, (vec_destroy_cb) ti_vtask_drop); + vec_destroy(collection->commits, (vec_destroy_cb) ti_commit_destroy); smap_destroy(collection->procedures, (smap_destroy_cb) ti_procedure_destroy); smap_destroy(collection->named_rooms, NULL); ti_types_destroy(collection->types); diff --git a/src/ti/commit.c b/src/ti/commit.c index 0d6d1da5..62f1106a 100644 --- a/src/ti/commit.c +++ b/src/ti/commit.c @@ -2,11 +2,13 @@ * ti/commit.c */ #include +#include #include +#include #include #include -ti_commit_t * ti_commit_create( +static ti_commit_t * commit_create( uint64_t id, time_t ts, const char * code, @@ -48,13 +50,36 @@ ti_commit_t * ti_commit_create( return commit; fail3: - ti_val_unsafe_drop(commit->by); + ti_val_unsafe_drop((ti_val_t *) commit->by); fail2: - ti_val_unsafe_drop(commit->message); + ti_val_unsafe_drop((ti_val_t *) commit->message); fail1: - ti_val_unsafe_drop(commit->code); + ti_val_unsafe_drop((ti_val_t *) commit->code); fail0: free(commit); + return NULL; +} + +ti_commit_t * ti_commit_from_up(mp_unp_t * up) +{ + mp_obj_t obj, mp_id, mp_ts, mp_code, mp_by, mp_message, mp_err_msg; + if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 6 || + mp_next(up, &mp_id) != MP_U64 || + mp_next(up, &mp_ts) != MP_U64 || + mp_next(up, &mp_code) != MP_STR || + mp_next(up, &mp_by) != MP_STR || + mp_next(up, &mp_message) != MP_STR || + (mp_next(up, &mp_err_msg) != MP_STR && mp_err_msg.tp == MP_NIL)) + return NULL; + return commit_create( + mp_id.via.u64, + (time_t) mp_ts.via.u64, + mp_code.via.str.data, mp_code.via.str.n, + mp_message.via.str.data, mp_message.via.str.n, + mp_by.via.str.data, mp_by.via.str.n, + mp_err_msg.tp == MP_STR ? mp_err_msg.via.str.data : NULL, + mp_err_msg.tp == MP_STR ? mp_err_msg.via.str.n : 0 + ); } ti_commit_t * ti_commit_make( @@ -87,24 +112,79 @@ void ti_commit_destroy(ti_commit_t * commit) { if (!commit) return; - ti_val_unsafe_drop(commit->code); - ti_val_unsafe_drop(commit->message); - ti_val_unsafe_drop(commit->by); - ti_val_drop(commit->err_msg); + ti_val_unsafe_drop((ti_val_t *) commit->code); + ti_val_unsafe_drop((ti_val_t *) commit->message); + ti_val_unsafe_drop((ti_val_t *) commit->by); + ti_val_drop((ti_val_t *) commit->err_msg); free(commit); } -int ti_commit_set_history(vec_t ** commits, _Bool state) +int ti_commit_to_pk(ti_commit_t * commit, msgpack_packer * pk) { - if (state) - { - if (*commits) - return 0; + return -( + msgpack_pack_array(pk, 6) || + msgpack_pack_uint64(pk, commit->id) || + msgpack_pack_uint64(pk, (uint64_t) commit->ts) || + mp_pack_strn(pk, commit->code->data, commit->code->n) || + mp_pack_strn(pk, commit->message->data, commit->message->n) || + mp_pack_strn(pk, commit->by->data, commit->by->n) || ( + commit->err_msg + ? mp_pack_strn(pk, commit->err_msg->data, commit->err_msg->n) + : msgpack_pack_nil(pk) + ) + ); +} + +int ti_commit_to_client_pk( + ti_commit_t * commit, + _Bool detail, + msgpack_packer * pk) +{ + const char * isotimestr = ti_datetime_ts_str((const time_t *) &commit->ts); + return -( + msgpack_pack_map(pk, 4 + !!detail + !!commit->err_msg) || + + mp_pack_str(pk, "id") || + msgpack_pack_uint64(pk, commit->id) || + + mp_pack_str(pk, "created_on") || + mp_pack_str(pk, isotimestr) || + + mp_pack_str(pk, "by") || + mp_pack_strn(pk, commit->by->data, commit->by->n) || + + mp_pack_str(pk, "message") || + mp_pack_strn(pk, commit->message, commit->message->n) || - *commits = vec_new(8); - return !!(*commits); + (detail && ( + mp_pack_str(pk, "code") || + mp_pack_strn(pk, commit->code, commit->code->n) + )) || + + (commit->err_msg && ( + mp_pack_str(pk, "err_msg") || + mp_pack_strn(pk, commit->err_msg, commit->err_msg->n) + )) + ); +} + +ti_val_t * ti_commit_as_mpval(ti_commit_t * commit, _Bool detail) +{ + ti_raw_t * raw; + msgpack_packer pk; + msgpack_sbuffer buffer; + + mp_sbuffer_alloc_init(&buffer, sizeof(ti_raw_t), sizeof(ti_raw_t)); + msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); + + if (ti_commit_to_client_pk(commit, detail, &pk)) + { + msgpack_sbuffer_destroy(&buffer); + return NULL; } - vec_destroy(*commits, ti_commit_destroy); - *commits = NULL; - return 0; -} \ No newline at end of file + + raw = (ti_raw_t *) buffer.data; + ti_raw_init(raw, TI_VAL_MPDATA, buffer.size); + + return (ti_val_t *) raw; +} diff --git a/src/ti/commits.c b/src/ti/commits.c index 822ccacb..71c051d3 100644 --- a/src/ti/commits.c +++ b/src/ti/commits.c @@ -2,47 +2,289 @@ * ti/commits.c */ #include +#include +#include #include typedef struct { - ti_raw_t * scope; - ti_raw_t * contains; - ti_regex_t * match; - ti_vint_t * id; - ti_vint_t * last; - ti_vint_t * first; - ti_vbool_t * detail; - ti_datetime_t * before; - ti_datetime_t * after; + ti_commits_options_t * options; ex_t * e; + _Bool allow_detail; } commits__options_t; +static int commits__options( + ti_raw_t * key, + ti_val_t * val, + commits__options_t * w) +{ + if (ti_raw_eq_strn(key, "scope", strlen("scope"))) + { + if (!ti_val_is_str(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `scope` to be of type `"TI_VAL_STR_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->scope = (ti_raw_t *) val; + return 0; + } + if (ti_raw_eq_strn(key, "contains", strlen("contains"))) + { + if (!ti_val_is_str(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `contains` to be of type `"TI_VAL_STR_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->contains = (ti_raw_t *) val; + return 0; + } + if (ti_raw_eq_strn(key, "match", strlen("match"))) + { + if (!ti_val_is_regex(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `match` to be of type `"TI_VAL_REGEX_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->match = (ti_regex_t *) val; + return 0; + } + if (ti_raw_eq_strn(key, "id", strlen("id"))) + { + if (!ti_val_is_int(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `id` to be of type `"TI_VAL_INT_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->id = (ti_vint_t *) val; + return 0; + } + if (ti_raw_eq_strn(key, "last", strlen("last"))) + { + if (!ti_val_is_int(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `last` to be of type `"TI_VAL_INT_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->last = (ti_vint_t *) val; + return 0; + } + if (ti_raw_eq_strn(key, "first", strlen("first"))) + { + if (!ti_val_is_int(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `first` to be of type `"TI_VAL_INT_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->first = (ti_vint_t *) val; + return 0; + } + if (ti_raw_eq_strn(key, "before", strlen("before"))) + { + if (!ti_val_is_datetime(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `before` to be of " + "type `"TI_VAL_DATETIME_S"` or `"TI_VAL_TIMEVAL_S"` " + "but got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->before = (ti_datetime_t *) val; + return 0; + } + if (ti_raw_eq_strn(key, "after", strlen("after"))) + { + if (!ti_val_is_datetime(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `after` to be of " + "type `"TI_VAL_DATETIME_S"` or `"TI_VAL_TIMEVAL_S"` " + "but got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->after = (ti_datetime_t *) val; + return 0; + } + if (w->allow_detail && ti_raw_eq_strn(key, "detail", strlen("detail"))) + { + if (!ti_val_is_bool(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `detail` to be of type `"TI_VAL_BOOL_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->detail = (ti_vbool_t *) val; + return 0; + } + ex_set(w->e, EX_VALUE_ERROR, + "invalid option `%.*s`; valid options: `scope`, `contains`, " + "`match`, `id`, `first`, `last`, `before`, `after`%s", + key->n, key->data, + w->allow_detail ? ", `detail`" : ""); + return 0; +} + +int ti_commits_options( + ti_commits_options_t * options, + ti_thing_t * thing, + _Bool allow_detail, + ex_t * e) +{ + commits__options_t w = { + .options=options, + .e=e, + .allow_detail=allow_detail, + }; + + if (ti_thing_walk(thing, (ti_thing_item_cb) commits__options, &w)) + return e->nr; /* error is set */ + + if (!options->contains && + !options->match && + !options->id && + !options->first && + !options->last && + !options->before && + !options->after) + ex_set(e, EX_LOOKUP_ERROR, + "at least one of the following options must be set: " + "`contains`, `match`, `id`, `first`, `last`, `before`, `after`"); + return e->nr; +} + +int ti_commits_history( + ti_commits_history_t * history, + ti_commits_options_t * options, + ex_t * e) + { + history->commits = &ti.commits; + history->access = &ti.access_thingsdb; + history->scope_id = 0; + + if (options->scope) + { + ti_scope_t scope; + if (ti_scope_init( + &scope, + (const char *) options->scope->data, + options->scope->n, + e)) + return e->nr; + + switch(scope.tp) + { + case TI_SCOPE_THINGSDB: + history->commits = &ti.commits; + history->access = &ti.access_thingsdb; + break; + case TI_SCOPE_NODE: + history->commits = NULL; + history->access = NULL; + break; + case TI_SCOPE_COLLECTION: + { + ti_collection_t * collection = ti_collections_get_by_strn( + scope.via.collection_name.name, + scope.via.collection_name.sz); + if (!collection) + { + ex_set(e, EX_LOOKUP_ERROR, + "collection `%.*s` not found", + scope.via.collection_name.sz, + scope.via.collection_name.name); + return e->nr; + } + history->commits = &collection->commits; + history->access = &collection->access; + history->scope_id = collection->id; + break; + } + } + if (!(*history->commits)) + ex_set(e, EX_OPERATION, + "history is not enabled for the `*.*s` scope", + options->scope->n, + options->scope->data); + return e->nr; + } + + for (vec_each(ti.collections->vec, ti_collection_t, collection)) + { + if (collection->commits) + { + if (*history->commits) + { + ex_set(e, EX_LOOKUP_ERROR, + "multiple scopes with `history` enabled; " + "use the `scope` option to specify an explicit scope"); + return e->nr; + } + history->commits = &collection->commits; + history->access = &collection->access; + history->scope_id = collection->id; + } + } + + if (!(*history->commits)) + ex_set(e, EX_LOOKUP_ERROR, + "no scope found with `history` enabled"DOC_SET_HISTORY); + return e->nr; +} + vec_t ** ti_commits_from_scope(ti_raw_t * scope, ex_t * e) { - if (ti_scope_init(&scope, (const char *) scope->data, scope->n, e)) + ti_scope_t scope_; + if (ti_scope_init(&scope_, (const char *) scope->data, scope->n, e)) return NULL; - switch(scope.tp) + switch(scope_.tp) { case TI_SCOPE_THINGSDB: return &ti.commits; case TI_SCOPE_NODE: return NULL; case TI_SCOPE_COLLECTION: - collection = ti_collections_get_by_strn( - scope.via.collection_name.name, - scope.via.collection_name.sz); + { + ti_collection_t * collection = ti_collections_get_by_strn( + scope_.via.collection_name.name, + scope_.via.collection_name.sz); if (collection) - return &collection->access; + return &collection->commits; ex_set(e, EX_LOOKUP_ERROR, "collection `%.*s` not found", - scope.via.collection_name.sz, - scope.via.collection_name.name); + scope_.via.collection_name.sz, + scope_.via.collection_name.name); + } } return NULL; } +void ti_commits_destroy(vec_t ** commits) +{ + vec_destroy(*commits, (vec_destroy_cb) ti_commit_destroy); + *commits = NULL; +} int ti_commits_set_history(vec_t ** commits, _Bool state) { if (state) @@ -53,7 +295,71 @@ int ti_commits_set_history(vec_t ** commits, _Bool state) *commits = vec_new(8); return !!(*commits); } - vec_destroy(*commits, ti_commit_destroy); - *commits = NULL; + ti_commits_destroy(commits); return 0; -} \ No newline at end of file +} + +static _Bool commits__match( + ti_commit_t * commit, + ti_commits_options_t * options) +{ + if (options->contains && ( + !ti_raw_contains(commit->code, options->contains) && + !ti_raw_contains(commit->message, options->contains) && + !ti_raw_contains(commit->by, options->contains) && ( + !commit->err_msg || + !ti_raw_contains(commit->err_msg, options->contains) + ) + )) + return false; + if (options->match && ( + !ti_regex_test(options->match, commit->code) && + !ti_regex_test(options->match, commit->message) && + !ti_regex_test(options->match, commit->by) && ( + !commit->err_msg || + !ti_regex_test(options->match, commit->err_msg) + ) + )) + return false; + if (options->id && commit->id != (uint64_t) VINT(options->id)) + return false; + + return true; +} + +vec_t * ti_commits_find(vec_t * commits, ti_commits_options_t * options) +{ + vec_t * vec = vec_new(8); + if (!vec) + return NULL; + for (vec_each(commits, ti_commit_t, commit)) + if (commits__match(commit, options) && vec_push(&vec, commit)) + goto fail; + + return vec; +fail: + vec_destroy(vec, NULL); + return NULL; +} + +vec_t * ti_commits_del(vec_t ** commits, ti_commits_options_t * options) +{ + vec_t * vec = vec_new(8); + uint32_t n = (*commits)->n; + if (!vec) + return NULL; + + for (vec_each_rev(*commits, ti_commit_t, commit) --n) + if (commits__match(commit, options) && + vec_push(&vec, vec_remove(*commits, n))) + goto fail; + + (void) vec_shrink(commits); + return vec; +fail: + /* revert */ + for (vec_each(vec, ti_commit_t, commit)) + VEC_push(*commits, commit); + vec_destroy(vec, NULL); + return NULL; +} diff --git a/src/ti/ctask.c b/src/ti/ctask.c index 1e51edcc..cc03ab0f 100644 --- a/src/ti/ctask.c +++ b/src/ti/ctask.c @@ -3286,61 +3286,33 @@ int ctask__arr_remove(ti_thing_t * thing, mp_unp_t * up) */ static int ctask__commit_add(ti_thing_t * thing, mp_unp_t * up) { - int rc; - mp_obj_t obj, mp_name, mp_created; - ti_collection_t * collection = thing->collection; - ti_procedure_t * procedure; - ti_closure_t * closure; - ti_vup_t vup = { - .isclient = false, - .collection = collection, - .up = up, - }; - - if (mp_next(up, &obj) != MP_MAP || obj.via.sz != 3 || - mp_skip(up) != MP_STR || - mp_next(up, &mp_name) != MP_STR || - mp_skip(up) != MP_STR || - mp_next(up, &mp_created) != MP_U64 || - mp_skip(up) != MP_STR) + vec_t ** commits = &thing->collection->commits; + ti_commit_t * commit = ti_commit_from_up(up); + if (!commit) { log_critical( - "task `new_procedure` for "TI_COLLECTION_ID": " - "missing map or name", - collection->id); + "failed to unpack commit from task `commit_add` " + "for the @collection:%.*s scope", + thing->collection->name->n, thing->collection->name->data); return -1; } - - closure = (ti_closure_t *) ti_val_from_vup(&vup); - procedure = NULL; - - if (!closure || !ti_val_is_closure((ti_val_t *) closure) || - !(procedure = ti_procedure_create( - mp_name.via.str.data, - mp_name.via.str.n, - closure, - mp_created.via.u64))) - goto failed; - - rc = ti_procedures_add(collection->procedures, procedure); - if (rc == 0) + if (!(*commits)) { - ti_decref(closure); - return 0; /* success */ + log_error( + "commits not enabled for the @collection:%.*s scope", + thing->collection->name->n, thing->collection->name->data); + goto fail; } - - if (rc < 0) - log_critical(EX_MEMORY_S); - else - log_critical( - "task `new_procedure` for "TI_COLLECTION_ID": " - "procedure `%s` already exists", - collection->id, - procedure->name->str); - -failed: - ti_procedure_destroy(procedure); - ti_val_drop((ti_val_t *) closure); + if (vec_push(commits, commit)) + { + log_error( + "failed to add commit for the @collection:%.*s scope", + thing->collection->name->n, thing->collection->name->data); + goto fail; + } + return 0; +fail: + ti_commit_destroy(commit); return -1; } @@ -3449,7 +3421,7 @@ int ti_ctask_run(ti_thing_t * thing, mp_unp_t * up) case TI_TASK_WHITELIST_ADD: break; case TI_TASK_WHITELIST_DEL: break; case TI_TASK_SET_HISTORY: break; - case TI_TASK_COMMIT_DEL: break; + case TI_TASK_DEL_HISTORY: break; case TI_TASK_COMMIT_ADD: return ctask__commit_add(thing, up); } diff --git a/src/ti/export.c b/src/ti/export.c index 2d191b0c..d898d120 100644 --- a/src/ti/export.c +++ b/src/ti/export.c @@ -651,7 +651,7 @@ static int export__commit(ti_fmt_t * fmt, ti_collection_t * collection) return buf_append_fmt( &fmt->buf, "try(commit('Source: collection `%.*s`'));\n\n", - collection->name->n, collection->name-data); + collection->name->n, collection->name->data); } static int export__collection(ti_fmt_t * fmt, ti_collection_t * collection) diff --git a/src/ti/qbind.c b/src/ti/qbind.c index c146c494..271604f2 100644 --- a/src/ti/qbind.c +++ b/src/ti/qbind.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +99,7 @@ #include #include #include +#include #include #include #include @@ -215,6 +218,7 @@ #include #include #include +#include #include #include #include @@ -302,11 +306,11 @@ static void qbind__statement(ti_qbind_t * qbind, cleri_node_t * nd); */ enum { - TOTAL_KEYWORDS = 277, + TOTAL_KEYWORDS = 281, MIN_WORD_LENGTH = 2, MAX_WORD_LENGTH = 17, - MIN_HASH_VALUE = 12, - MAX_HASH_VALUE = 751 + MIN_HASH_VALUE = 38, + MAX_HASH_VALUE = 796 }; /* @@ -318,32 +322,32 @@ static inline unsigned int qbind__hash( { static unsigned short asso_values[] = { - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 4, 3, - 6, 752, 3, 752, 3, 752, 4, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 2, 752, 19, 120, 85, - 23, 4, 136, 275, 180, 2, 6, 71, 24, 59, - 11, 41, 110, 44, 5, 2, 3, 28, 281, 191, - 227, 252, 25, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752, 752, 752, 752, 752, - 752, 752, 752, 752, 752, 752 + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 16, 15, + 18, 797, 19, 797, 18, 797, 16, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 15, 797, 20, 58, 82, + 24, 17, 152, 371, 112, 15, 18, 73, 17, 25, + 20, 22, 131, 15, 16, 15, 16, 52, 174, 300, + 204, 140, 29, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797, 797, 797, 797, 797, + 797, 797, 797, 797, 797, 797 }; register unsigned int hval = n; @@ -517,6 +521,7 @@ qbind__fmap_t qbind__fn_mapping[TOTAL_KEYWORDS] = { {.name="code", .fn=do__f_code, CHAIN_NE}, {.name="collection_info", .fn=do__f_collection_info, ROOT_NE}, {.name="collections_info", .fn=do__f_collections_info, ROOT_NE}, + {.name="commit", .fn=do__f_commit, ROOT_NE}, {.name="contains", .fn=do__f_contains, CHAIN_NE}, {.name="copy", .fn=do__f_copy, CHAIN_NE}, {.name="cos", .fn=do__f_cos, ROOT_NE}, @@ -529,6 +534,7 @@ qbind__fmap_t qbind__fn_mapping[TOTAL_KEYWORDS] = { {.name="del_collection", .fn=do__f_del_collection, ROOT_TE}, {.name="del_enum", .fn=do__f_del_enum, ROOT_CE}, {.name="del_expired", .fn=do__f_del_expired, ROOT_TE}, + {.name="del_history", .fn=do__f_del_history, ROOT_TE}, {.name="del_module", .fn=do__f_del_module, ROOT_TE}, {.name="del_node", .fn=do__f_del_node, ROOT_TE}, {.name="del_procedure", .fn=do__f_del_procedure, ROOT_BE}, @@ -577,6 +583,7 @@ qbind__fmap_t qbind__fn_mapping[TOTAL_KEYWORDS] = { {.name="has_token", .fn=do__f_has_token, ROOT_NE}, {.name="has_type", .fn=do__f_has_type, ROOT_NE}, {.name="has_user", .fn=do__f_has_user, ROOT_NE}, + {.name="history", .fn=do__f_history, ROOT_NE}, {.name="id", .fn=do__f_id, CHAIN_NE}, {.name="import", .fn=do__f_import, ROOT_CE}, {.name="index_of", .fn=do__f_index_of, CHAIN_NE}, @@ -701,6 +708,7 @@ qbind__fmap_t qbind__fn_mapping[TOTAL_KEYWORDS] = { {.name="set_closure", .fn=do__f_set_closure, CHAIN_BE}, {.name="set_default_deep", .fn=do__f_set_default_deep, ROOT_TE}, {.name="set_enum", .fn=do__f_set_enum, ROOT_CE}, + {.name="set_history", .fn=do__f_set_history, ROOT_TE}, {.name="set_log_level", .fn=do__f_set_log_level, ROOT_NE}, {.name="set_module_conf", .fn=do__f_set_module_conf, ROOT_TE}, {.name="set_module_scope", .fn=do__f_set_module_scope, ROOT_TE}, @@ -730,9 +738,9 @@ qbind__fmap_t qbind__fn_mapping[TOTAL_KEYWORDS] = { {.name="time_zones_info", .fn=do__f_time_zones_info, ROOT_NE}, {.name="timeit", .fn=do__f_timeit, ROOT_NE}, {.name="timeval", .fn=do__f_timeval, ROOT_NE}, + {.name="to", .fn=do__f_to, CHAIN_NE}, {.name="to_thing", .fn=do__f_to_thing, CHAIN_CE_X}, {.name="to_type", .fn=do__f_to_type, CHAIN_CE_X}, - {.name="to", .fn=do__f_to, CHAIN_NE}, {.name="trim", .fn=do__f_trim, CHAIN_NE}, {.name="trim_left", .fn=do__f_trim_left, CHAIN_NE}, {.name="trim_right", .fn=do__f_trim_right, CHAIN_NE}, @@ -750,8 +758,8 @@ qbind__fmap_t qbind__fn_mapping[TOTAL_KEYWORDS] = { {.name="upper", .fn=do__f_upper, CHAIN_NE}, {.name="user_info", .fn=do__f_user_info, ROOT_NE}, {.name="users_info", .fn=do__f_users_info, ROOT_NE}, - {.name="value_err", .fn=do__f_value_err, ROOT_NE}, {.name="value", .fn=do__f_value, CHAIN_NE}, + {.name="value_err", .fn=do__f_value_err, ROOT_NE}, {.name="values", .fn=do__f_values, CHAIN_NE}, {.name="vmap", .fn=do__f_vmap, CHAIN_NE}, {.name="week", .fn=do__f_week, CHAIN_NE}, diff --git a/src/ti/store.c b/src/ti/store.c index 52248e54..0c4057ad 100644 --- a/src/ti/store.c +++ b/src/ti/store.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ static const char * store__tmp_path = "__tmp/"; static const char * store__access_node_fn = "access_node.mp"; static const char * store__access_thingsdb_fn = "access_thingsdb.mp"; static const char * store__collections_fn = "collections.mp"; +static const char * store__commits_fn = "commits.mp"; static const char * store__id_stat_fn = "idstat.mp"; static const char * store__names_fn = "names.mp"; static const char * store__procedures_fn = "procedures.mp"; @@ -58,6 +60,7 @@ static void store__set_filename(_Bool use_tmp) memcpy(store->access_node_fn + store->fn_offset, path, n); memcpy(store->access_thingsdb_fn + store->fn_offset, path, n); memcpy(store->collections_fn + store->fn_offset, path, n); + memcpy(store->commits_fn + store->fn_offset, path, n); memcpy(store->id_stat_fn + store->fn_offset, path, n); memcpy(store->names_fn + store->fn_offset, path, n); memcpy(store->procedures_fn + store->fn_offset, path, n); @@ -112,6 +115,7 @@ int ti_store_create(void) store->collections_fn = fx_path_join( store->tmp_path, store__collections_fn); + store->commits_fn = fx_path_join(store->tmp_path, store__commits_fn); store->id_stat_fn = fx_path_join(store->tmp_path, store__id_stat_fn); store->names_fn = fx_path_join(store->tmp_path, store__names_fn); store->procedures_fn = fx_path_join(store->tmp_path, store__procedures_fn); @@ -126,6 +130,7 @@ int ti_store_create(void) !store->access_node_fn || !store->access_thingsdb_fn || !store->collections_fn || + !store->commits_fn || !store->id_stat_fn || !store->names_fn || !store->procedures_fn || @@ -170,6 +175,7 @@ void ti_store_destroy(void) free(store->access_node_fn); free(store->access_thingsdb_fn); free(store->collections_fn); + free(store->commits_fn); free(store->id_stat_fn); free(store->names_fn); free(store->procedures_fn); @@ -212,7 +218,8 @@ int ti_store_store(void) ti_store_collections_store(store->collections_fn) || ti_store_procedures_store(ti.procedures, store->procedures_fn) || ti_store_tasks_store(ti.tasks->vtasks, store->tasks_fn) || - ti_store_modules_store(store->modules_fn)) + ti_store_modules_store(store->modules_fn) || + ti_store_commits_store(ti.commits, store->commits_fn)) goto failed; for (vec_each(ti.collections->vec, ti_collection_t, collection)) @@ -267,7 +274,10 @@ int ti_store_store(void) store_collection->procedures_fn) || ti_store_tasks_store( collection->vtasks, - store_collection->tasks_fn) + store_collection->tasks_fn) || + ti_store_commits_store( + collection->commits, + store_collection->commits_fn) ); } ti_store_collection_destroy(store_collection); @@ -348,7 +358,8 @@ int ti_store_restore(void) &ti.tasks->vtasks, store->tasks_fn, NULL) || - ti_store_modules_restore(store->modules_fn)); + ti_store_modules_restore(store->modules_fn) || + ti_store_commits_restore(&ti.commits, store->commits_fn)); if (rc) goto stop; @@ -430,7 +441,10 @@ int ti_store_restore(void) ti_store_procedures_restore( collection->procedures, store_collection->procedures_fn, - collection) + collection) || + ti_store_commits_restore( + &collection->commits, + store_collection->commits_fn) ); ti_store_collection_destroy(store_collection); diff --git a/src/ti/store/storecollection.c b/src/ti/store/storecollection.c index 28fa11bd..03348970 100644 --- a/src/ti/store/storecollection.c +++ b/src/ti/store/storecollection.c @@ -9,16 +9,17 @@ #include static const char * collection___access_fn = "access.mp"; +static const char * collection___commits_fn = "commits.mp"; static const char * collection___dat_fn = "collection.dat"; +static const char * collection___enums_fn = "enums.mp"; +static const char * collection___gcprops_fn = "gcprops.mp"; +static const char * collection___gcthings_fn = "gcthings.mp"; +static const char * collection___named_rooms_fn = "named_rooms.mp"; static const char * collection___procedures_fn = "procedures.mp"; -static const char * collection___tasks_fn = "tasks.mp"; static const char * collection___props_fn = "props.mp"; -static const char * collection___named_rooms_fn = "named_rooms.mp"; +static const char * collection___tasks_fn = "tasks.mp"; static const char * collection___things_fn = "things.mp"; static const char * collection___types_fn = "types.mp"; -static const char * collection___enums_fn = "enums.mp"; -static const char * collection___gcprops_fn = "gcprops.mp"; -static const char * collection___gcthings_fn = "gcthings.mp"; ti_store_collection_t * ti_store_collection_create( const char * path, @@ -40,6 +41,7 @@ ti_store_collection_t * ti_store_collection_create( store_collection->access_fn = fx_path_join(cpath, collection___access_fn); store_collection->collection_fn = fx_path_join(cpath, collection___dat_fn); + store_collection->commits_fn = fx_path_join(cpath, collection___commits_fn); store_collection->procedures_fn = fx_path_join(cpath, collection___procedures_fn); store_collection->tasks_fn = fx_path_join(cpath, collection___tasks_fn); store_collection->props_fn = fx_path_join(cpath, collection___props_fn); @@ -52,6 +54,7 @@ ti_store_collection_t * ti_store_collection_create( if ( !store_collection->access_fn || !store_collection->collection_fn || + !store_collection->commits_fn || !store_collection->procedures_fn || !store_collection->tasks_fn || !store_collection->props_fn || @@ -79,6 +82,7 @@ void ti_store_collection_destroy(ti_store_collection_t * store_collection) free(store_collection->access_fn); free(store_collection->collection_fn); free(store_collection->collection_path); + free(store_collection->commits_fn); free(store_collection->procedures_fn); free(store_collection->tasks_fn); free(store_collection->props_fn); @@ -309,3 +313,15 @@ char * ti_store_collection_gcthings_fn( free(cpath); return fn; } + +char * ti_store_collection_commits_fn( + const char * path, + uint64_t collection_id) +{ + char * fn, * cpath = ti_store_collection_get_path(path, collection_id); + if (!cpath) + return NULL; + fn = fx_path_join(cpath, collection___commits_fn); + free(cpath); + return fn; +} \ No newline at end of file diff --git a/src/ti/store/storecommits.c b/src/ti/store/storecommits.c new file mode 100644 index 00000000..c7e1006f --- /dev/null +++ b/src/ti/store/storecommits.c @@ -0,0 +1,128 @@ +/* + * ti/store/storecommits.c + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +int ti_store_commits_store(vec_t * commits, const char * fn) +{ + msgpack_packer pk; + FILE * f = fopen(fn, "w"); + if (!f) + { + log_errno_file("cannot open file", errno, fn); + return -1; + } + + msgpack_packer_init(&pk, f, msgpack_fbuffer_write); + + if (!commits) + { + if (msgpack_pack_map(&pk, 0)) + goto fail; + } + else + { + if (msgpack_pack_map(&pk, 1) || + mp_pack_str(&pk, "commits") || + msgpack_pack_array(&pk, commits->n) + ) goto fail; + + for (vec_each(commits, ti_commit_t, commit)) + if (ti_commit_to_pk(commit, &pk)) + goto fail; + } + + log_debug("stored commits to file: `%s`", fn); + goto done; +fail: + log_error("failed to write file: `%s`", fn); +done: + if (fclose(f)) + { + log_errno_file("cannot close file", errno, fn); + return -1; + } + (void) ti_sleep(5); + return 0; +} + +int ti_store_commits_restore(vec_t ** commits, const char * fn) +{ + int rc = -1; + fx_mmap_t fmap; + size_t i; + mp_obj_t obj; + mp_unp_t up; + ti_commit_t * commit; + + /* clear existing commits (may exist in the thingsdb scope) */ + ti_commits_destroy(commits); + + if (!fx_file_exist(fn)) + { + /* + * TODO: (COMPAT) This check may be removed when we no longer require + * backwards compatibility with previous versions of ThingsDB + * where the commits file did not exist. + */ + log_warning( + "no commits found; " + "file `%s` is missing", + fn); + return ti_store_commits_store(NULL, fn); + } + + fx_mmap_init(&fmap, fn); + if (fx_mmap_open(&fmap)) /* fx_mmap_open() is a log function */ + goto fail0; + + mp_unp_init(&up, fmap.data, fmap.n); + + if (mp_next(&up, &obj) != MP_MAP || obj.via.sz > 1) + goto fail1; + + if (obj.via.sz == 0) + { + rc = 0; + goto done; /* done, commits disabled */ + } + + if (ti_commits_set_history(commits, true)) + goto fail1; + + for (i = obj.via.sz; i--;) + { + commit = ti_commit_from_up(&up); + if (!commit) + goto fail1; + + if (vec_push(commits, commit)) + goto fail2; + } + + rc = 0; + goto done; + +fail2: + ti_commit_destroy(commit); + +done: +fail1: + if (fx_mmap_close(&fmap)) + rc = -1; +fail0: + if (rc) + log_critical("failed to restore from file: `%s`", fn); + + return rc; +} diff --git a/src/ti/syncfull.c b/src/ti/syncfull.c index 9eecac0d..3c4496dc 100644 --- a/src/ti/syncfull.c +++ b/src/ti/syncfull.c @@ -30,6 +30,7 @@ typedef enum SYNCFULL__COLLECTIONS_FILE, SYNCFULL__MODULES_FILE, SYNCFULL__ID_STAT_FILE, + SYNCFULL__COMMITS_FILE, /* collection files */ SYNCFULL__COLLECTION_DAT_FILE, SYNCFULL__COLLECTION_TYPES_FILE, @@ -42,6 +43,7 @@ typedef enum SYNCFULL__COLLECTION_THINGS_FILE, SYNCFULL__COLLECTION_PROPS_FILE, SYNCFULL__COLLECTION_NAMED_ROOMS_FILE, + SYNCFULL__COLLECTION_COMMITS_FILE, /* end */ SYNCFULL__COLLECTION_END, } syncfull__file_t; @@ -70,6 +72,8 @@ static char * syncfull__get_fn(uint64_t scope_id, syncfull__file_t ft) return strdup(ti.store->modules_fn); case SYNCFULL__ID_STAT_FILE: return strdup(ti.store->id_stat_fn); + case SYNCFULL__COMMITS_FILE: + return strdup(ti.store->commits_fn); case SYNCFULL__COLLECTION_DAT_FILE: return ti_store_collection_dat_fn(path, scope_id); case SYNCFULL__COLLECTION_TYPES_FILE: @@ -92,6 +96,8 @@ static char * syncfull__get_fn(uint64_t scope_id, syncfull__file_t ft) return ti_store_collection_props_fn(path, scope_id); case SYNCFULL__COLLECTION_NAMED_ROOMS_FILE: return ti_store_collection_named_rooms_fn(path, scope_id); + case SYNCFULL__COLLECTION_COMMITS_FILE: + return ti_store_collection_commits_fn(path, scope_id); case SYNCFULL__COLLECTION_END: break; diff --git a/src/ti/task.c b/src/ti/task.c index dae617a6..f17cc807 100644 --- a/src/ti/task.c +++ b/src/ti/task.c @@ -3142,8 +3142,12 @@ int ti_task_add_whitelist_del( int ti_task_add_commit_add(ti_task_t * task, ti_commit_t * commit) { - size_t alloc = (50 + commit->code->n + commit->info->n + commit->by->n + ( - commit->err_msg ? commit->err_msg->n : 0) + size_t alloc = ( + 50 + + commit->code->n + + commit->message->n + + commit->by->n + + (commit->err_msg ? commit->err_msg->n : 0) ); ti_data_t * data; msgpack_packer pk; @@ -3153,18 +3157,85 @@ int ti_task_add_commit_add(ti_task_t * task, ti_commit_t * commit) return -1; msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); - msgpack_pack_array(pk, 2); + msgpack_pack_array(&pk, 2); + + msgpack_pack_uint8(&pk, TI_TASK_COMMIT_ADD); + (void) ti_commit_to_pk(commit, &pk); + + data = (ti_data_t *) buffer.data; + ti_data_init(data, buffer.size); + + if (vec_push(&task->list, data)) + goto fail_data; + + task__upd_approx_sz(task, data); + return 0; + +fail_data: + free(data); + return -1; +} + +int ti_task_add_set_history(ti_task_t * task, uint64_t scope_id, _Bool state) +{ + size_t alloc = 40; + ti_data_t * data; + msgpack_packer pk; + msgpack_sbuffer buffer; + + if (mp_sbuffer_alloc_init(&buffer, alloc, sizeof(ti_data_t))) + return -1; + msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); + + msgpack_pack_array(&pk, 2); + + msgpack_pack_uint8(&pk, TI_TASK_SET_HISTORY); + msgpack_pack_array(&pk, 2); + + msgpack_pack_uint64(&pk, scope_id); + mp_pack_bool(&pk, state); + + data = (ti_data_t *) buffer.data; + ti_data_init(data, buffer.size); + + if (vec_push(&task->list, data)) + goto fail_data; + + task__upd_approx_sz(task, data); + return 0; + +fail_data: + free(data); + return -1; +} + +int ti_task_add_del_history( + ti_task_t * task, + uint64_t scope_id, + vec_t * commits) +{ + size_t alloc = 28 + (commits->n * 9); + ti_data_t * data; + msgpack_packer pk; + msgpack_sbuffer buffer; + + if (mp_sbuffer_alloc_init(&buffer, alloc, sizeof(ti_data_t))) + return -1; + msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); - msgpack_pack_uint8(pk, TI_TASK_COMMIT_ADD); - msgpack_pack_array(pk, 5 + !!commit->err_msg); + msgpack_pack_array(&pk, 2); + + msgpack_pack_uint8(&pk, TI_TASK_DEL_HISTORY); + msgpack_pack_array(&pk, 2); + + msgpack_pack_uint64(&pk, scope_id); + msgpack_pack_array(&pk, commits->n); + + for (vec_each(commits, ti_commit_t, commit)) + msgpack_pack_uint64(&pk, commit->id); - msgpack_pack_uint64(pk, commit->id); - msgpack_pack_uint64(pk, (uint64_t) commit->ts); - mp_pack_strn(pk, commit->code->data, commit->code->n); - mp_pack_strn(pk, commit->message->data, commit->message->n); - mp_pack_strn(pk, commit->by->data, commit->by->n); - if (commit->err_msg) - mp_pack_strn(pk, commit->err_msg->data, commit->err_msg->n); + data = (ti_data_t *) buffer.data; + ti_data_init(data, buffer.size); if (vec_push(&task->list, data)) goto fail_data; diff --git a/src/ti/ttask.c b/src/ti/ttask.c index e43a4199..a3fbc8a6 100644 --- a/src/ti/ttask.c +++ b/src/ti/ttask.c @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -703,12 +705,12 @@ static int ttask__whitelist_del(mp_unp_t * up) return 0; } -static int ttask__set_historyl(mp_unp_t * up) +static int ttask__set_history(mp_unp_t * up) { vec_t ** commits; mp_obj_t obj, mp_scope, mp_state; - if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 2 + if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 2 || mp_next(up, &mp_scope) != MP_U64 || mp_next(up, &mp_state) != MP_BOOL) { @@ -735,7 +737,7 @@ static int ttask__set_historyl(mp_unp_t * up) commits = &ti.commits; } - if (ti_commit_set_history(&commits, mp_state.via.bool_)) + if (ti_commits_set_history(commits, mp_state.via.bool_)) { log_critical("failed to set `set_history`"); return -1; @@ -744,6 +746,89 @@ static int ttask__set_historyl(mp_unp_t * up) return 0; } +static int ttask__del_history(mp_unp_t * up) +{ + vec_t ** commits; + mp_obj_t obj, mp_scope, mp_id; + size_t i; + + if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 2 || + mp_next(up, &mp_scope) != MP_U64 || + mp_next(up, &obj) != MP_ARR) + { + log_critical("task `del_history`: invalid task data"); + return -1; + } + + if (mp_scope.via.u64 > 1) + { + ti_collection_t * collection = \ + ti_collections_get_by_id(mp_scope.via.u64); + + if (!collection) + { + log_critical( + "task `set_history`: "TI_COLLECTION_ID" not found", + mp_scope.via.u64); + return -1; + } + commits = &collection->commits; + } + else + { + commits = &ti.commits; + } + + for (i = obj.via.sz; i--;) + { + uint32_t j = 0; + if (mp_next(up, &mp_id) != MP_U64) + { + log_critical("task `del_history`: invalid task data"); + return -1; + } + + for (vec_each(*commits, ti_commit_t, commit), j++) + { + if (commit->id == mp_id.via.u64) + { + ti_commit_destroy(vec_remove(*commits, j)); + break; + } + } + } + + (void) vec_shrink(commits); + return 0; +} + +static int ttask__commit_add(mp_unp_t * up) +{ + vec_t ** commits = &ti.commits; + ti_commit_t * commit = ti_commit_from_up(up); + if (!commit) + { + log_critical( + "failed to unpack commit from task `commit_add` " + "on the @thingsdb scope"); + return -1; + } + if (!(*commits)) + { + log_error("commits not enabled for @thingsdb scope"); + goto fail; + } + if (vec_push(commits, commit)) + { + log_error("failed to add commit for @thingsdb scope"); + goto fail; + } + return 0; +fail: + ti_commit_destroy(commit); + return -1; +} + /* * Returns 0 on success * - for example: {'id': id, 'key': value}, 'expire_ts': ts, 'description':..} @@ -1777,7 +1862,7 @@ int ti_ttask_run(ti_change_t * change, mp_unp_t * up) case TI_TASK_WHITELIST_ADD: return ttask__whitelist_add(up); case TI_TASK_WHITELIST_DEL: return ttask__whitelist_del(up); case TI_TASK_SET_HISTORY: return ttask__set_history(up); - case TI_TASK_COMMIT_DEL: return ttask__commit_del(up); + case TI_TASK_DEL_HISTORY: return ttask__del_history(up); case TI_TASK_COMMIT_ADD: return ttask__commit_add(up); } From 35dd3d9070fb2172d22e4ab575baeefb7d86c883 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Tue, 4 Nov 2025 17:55:51 +0100 Subject: [PATCH 08/23] build --- CHANGELOG.md | 5 +++++ CMakeLists.txt | 2 +- inc/ti/collection.h | 1 + inc/ti/collection.inline.h | 35 ----------------------------------- inc/ti/fn/fn.h | 15 ++++----------- inc/ti/fn/fnmodtype.h | 2 +- inc/ti/names.h | 14 ++++++-------- inc/ti/version.h | 2 +- itest/test_advanced.py | 3 +-- src/ti/collection.c | 33 +++++++++++++++++++++++++++++++++ src/ti/fn.c | 11 +++++++++++ src/ti/names.c | 10 ++++++++++ 12 files changed, 74 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bef8d8ac..e58297ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ * Include export enumerator members of type `thing`, pr #420. * Fixed missing "ID" (`#`) field in export, issue #421. * Added `type_all()` function, pr #422. +* Added commit history with new functions, pr#423. + - `commit()`: https://docs.thingsdb.io/v1/collection-api/commit/ + - `history()`: https://docs.thingsdb.io/v1/thingsdb-api/history/ + - `set_history()`: https://docs.thingsdb.io/v1/thingsdb-api/set_history/ + - `del_history()`: https://docs.thingsdb.io/v1/thingsdb-api/del_history/ # v1.7.6 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2923ff8e..8e21e6e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,7 +48,7 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") ) else() # Set these flags for other than MacOS build - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -finline-limit=4000") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -finline-limit=6000") if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2") if($ENV{LEGACY}) diff --git a/inc/ti/collection.h b/inc/ti/collection.h index 15073503..24f97864 100644 --- a/inc/ti/collection.h +++ b/inc/ti/collection.h @@ -25,6 +25,7 @@ ti_collection_t * ti_collection_create( uint8_t deep); void ti_collection_destroy(ti_collection_t * collection); void ti_collection_drop(ti_collection_t * collection); +int ti_collection_to_pk(ti_collection_t * collection, msgpack_packer * pk); _Bool ti_collection_name_check(const char * name, size_t n, ex_t * e); int ti_collection_rename( ti_collection_t * collection, diff --git a/inc/ti/collection.inline.h b/inc/ti/collection.inline.h index 93dd38d9..b33474b9 100644 --- a/inc/ti/collection.inline.h +++ b/inc/ti/collection.inline.h @@ -8,41 +8,6 @@ #include #include -static inline int ti_collection_to_pk( - ti_collection_t * collection, - msgpack_packer * pk) -{ - return -( - msgpack_pack_map(pk, 8) || - - mp_pack_str(pk, "collection_id") || - msgpack_pack_uint64(pk, collection->id) || - - mp_pack_str(pk, "name") || - mp_pack_strn(pk, collection->name->data, collection->name->n) || - - mp_pack_str(pk, "created_at") || - msgpack_pack_uint64(pk, collection->created_at) || - - mp_pack_str(pk, "things") || - msgpack_pack_uint64(pk, collection->things->n + collection->gc->n) || - - mp_pack_str(pk, "time_zone") || - mp_pack_strn(pk, collection->tz->name, collection->tz->n) || - - mp_pack_str(pk, "default_deep") || - msgpack_pack_uint64(pk, collection->deep) || - - mp_pack_str(pk, "next_free_id") || - msgpack_pack_uint64(pk, collection->next_free_id) || - - mp_pack_str(pk, "commit_history") || - (collection->commits - ? msgpack_pack_uint32(pk, collection->commits->n) - : mp_pack_str(pk, "disabled")) - ); -} - /* * Return a thing with a borrowed reference from the collection, or, * if not found, tries to restore the thing from the garbage collector. diff --git a/inc/ti/fn/fn.h b/inc/ti/fn/fn.h index 65b839fa..cba22427 100644 --- a/inc/ti/fn/fn.h +++ b/inc/ti/fn/fn.h @@ -105,6 +105,8 @@ int fn_arg_str_slow( ti_val_t * val, ex_t * e); +int fn_commit(const char * name, ti_query_t * query, ex_t * e); + static inline int fn_get_nargs(cleri_node_t * nd) { return (int) ((intptr_t) nd->data); @@ -445,17 +447,6 @@ static inline int fn_arg_name_check( return e->nr; } -static inline int fn_commit(const char * name, ti_query_t * query, ex_t * e) -{ - if (ti_query_commits(query) && !query->commit) - ex_set(e, EX_OPERATION, - "function `%s` requires a commit " - "before it can be used in the `%s` scope"DOC_COMMIT, - name, - ti_query_scope_name(query)); - return e->nr; -} - static inline int fn_not_collection_scope( const char * name, ti_query_t * query, @@ -487,6 +478,8 @@ static inline int fn_not_thingsdb_or_collection_scope( return e->nr; } + + static int fn_call(ti_query_t * query, cleri_node_t * nd, ex_t * e) { cleri_node_t * child = nd->children; /* first in argument list */ diff --git a/inc/ti/fn/fnmodtype.h b/inc/ti/fn/fnmodtype.h index a6ddb551..90659345 100644 --- a/inc/ti/fn/fnmodtype.h +++ b/inc/ti/fn/fnmodtype.h @@ -1547,7 +1547,7 @@ static int do__f_mod_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) fn_arg_name_check("mod_type", DOC_MOD_TYPE, 3, query->rval, e)) goto fail1; - name = ti_names_from_raw((ti_raw_t *) query->rval); + name = ti_names_from_raw_slow((ti_raw_t *) query->rval); if (!name) { ex_set_mem(e); diff --git a/inc/ti/names.h b/inc/ti/names.h index 4977cb8b..9bc8b76e 100644 --- a/inc/ti/names.h +++ b/inc/ti/names.h @@ -38,15 +38,13 @@ static inline ti_name_t * ti_names_get(const char * str, size_t n) return ti_names_new(str, n); } -static inline ti_name_t * ti_names_from_str(const char * str) -{ - return ti_names_get(str, strlen(str)); -} +ti_name_t * ti_names_get_slow(const char * str, size_t n); -static inline ti_name_t * ti_names_from_raw(ti_raw_t * raw) -{ - return ti_names_get((const char *) raw->data, raw->n); -} +#define ti_names_from_str(str__) ti_names_get((str__), strlen(str__)) +#define ti_names_from_raw(raw__) \ + ti_names_get((const char *) (raw__)->data, (raw__)->n) +#define ti_names_from_raw_slow(raw__) \ + ti_names_get_slow((const char *) (raw__)->data, (raw__)->n) static inline ti_name_t * ti_names_weak_from_raw(ti_raw_t * raw) { diff --git a/inc/ti/version.h b/inc/ti/version.h index a3025d8a..39d28e4a 100644 --- a/inc/ti/version.h +++ b/inc/ti/version.h @@ -25,7 +25,7 @@ * "-rc0" * "" */ -#define TI_VERSION_PRE_RELEASE "-alpha6" +#define TI_VERSION_PRE_RELEASE "-alpha7" #define TI_MAINTAINER \ "Jeroen van der Heijden " diff --git a/itest/test_advanced.py b/itest/test_advanced.py index 3de9378f..8af27960 100755 --- a/itest/test_advanced.py +++ b/itest/test_advanced.py @@ -339,8 +339,7 @@ async def test_wpo(self, client): with self.assertRaisesRegex( OperationError, - r'type `_Foo1` is dependent on type `_Name` by field `wrap` ' - r'with `wrap-only` mode enabled'): + r'xxx'): await client.query(r''' mod_type('_Foo1', 'wpo', false); ''') diff --git a/src/ti/collection.c b/src/ti/collection.c index a0c22bad..c6b1f41c 100644 --- a/src/ti/collection.c +++ b/src/ti/collection.c @@ -128,6 +128,39 @@ void ti_collection_drop(ti_collection_t * collection) log_critical(EX_MEMORY_S); } +int ti_collection_to_pk(ti_collection_t * collection, msgpack_packer * pk) +{ + return -( + msgpack_pack_map(pk, 8) || + + mp_pack_str(pk, "collection_id") || + msgpack_pack_uint64(pk, collection->id) || + + mp_pack_str(pk, "name") || + mp_pack_strn(pk, collection->name->data, collection->name->n) || + + mp_pack_str(pk, "created_at") || + msgpack_pack_uint64(pk, collection->created_at) || + + mp_pack_str(pk, "things") || + msgpack_pack_uint64(pk, collection->things->n + collection->gc->n) || + + mp_pack_str(pk, "time_zone") || + mp_pack_strn(pk, collection->tz->name, collection->tz->n) || + + mp_pack_str(pk, "default_deep") || + msgpack_pack_uint64(pk, collection->deep) || + + mp_pack_str(pk, "next_free_id") || + msgpack_pack_uint64(pk, collection->next_free_id) || + + mp_pack_str(pk, "commit_history") || + (collection->commits + ? msgpack_pack_uint32(pk, collection->commits->n) + : mp_pack_str(pk, "disabled")) + ); +} + _Bool ti_collection_name_check(const char * name, size_t n, ex_t * e) { if (n < ti_collection_min_name || n >= ti_collection_max_name) diff --git a/src/ti/fn.c b/src/ti/fn.c index 3ef1c737..1b38778f 100644 --- a/src/ti/fn.c +++ b/src/ti/fn.c @@ -14,3 +14,14 @@ int fn_arg_str_slow( name, argn, ti_val_str(val), doc); return e->nr; } + +int fn_commit(const char * name, ti_query_t * query, ex_t * e) +{ + if (ti_query_commits(query) && !query->commit) + ex_set(e, EX_OPERATION, + "function `%s` requires a commit " + "before it can be used in the `%s` scope"DOC_COMMIT, + name, + ti_query_scope_name(query)); + return e->nr; +} \ No newline at end of file diff --git a/src/ti/names.c b/src/ti/names.c index 38ba889c..53f21944 100644 --- a/src/ti/names.c +++ b/src/ti/names.c @@ -145,3 +145,13 @@ ti_name_t * ti_names_new(const char * str, size_t n) return name; } +ti_name_t * ti_names_get_slow(const char * str, size_t n) +{ + ti_name_t * name = smap_getn(names, str, n); + if (name) + { + ti_incref(name); + return name; + } + return ti_names_new(str, n); +} From 4a7dece2e3c70294f944409eff23a4ef98afe20e Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Tue, 4 Nov 2025 18:47:40 +0100 Subject: [PATCH 09/23] inline --- inc/ti/fn/fn.h | 13 ++++++++++--- itest/test_advanced.py | 5 ++++- src/ti/collection.c | 1 + src/ti/fn.c | 11 ----------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/inc/ti/fn/fn.h b/inc/ti/fn/fn.h index cba22427..eb4f3cfc 100644 --- a/inc/ti/fn/fn.h +++ b/inc/ti/fn/fn.h @@ -105,8 +105,6 @@ int fn_arg_str_slow( ti_val_t * val, ex_t * e); -int fn_commit(const char * name, ti_query_t * query, ex_t * e); - static inline int fn_get_nargs(cleri_node_t * nd) { return (int) ((intptr_t) nd->data); @@ -478,7 +476,16 @@ static inline int fn_not_thingsdb_or_collection_scope( return e->nr; } - +static inline int fn_commit(const char * name, ti_query_t * query, ex_t * e) +{ + if (ti_query_commits(query) && !query->commit) + ex_set(e, EX_OPERATION, + "function `%s` requires a commit " + "before it can be used in the `%s` scope"DOC_COMMIT, + name, + ti_query_scope_name(query)); + return e->nr; +} static int fn_call(ti_query_t * query, cleri_node_t * nd, ex_t * e) { diff --git a/itest/test_advanced.py b/itest/test_advanced.py index 8af27960..feb1c513 100755 --- a/itest/test_advanced.py +++ b/itest/test_advanced.py @@ -339,7 +339,8 @@ async def test_wpo(self, client): with self.assertRaisesRegex( OperationError, - r'xxx'): + r'type `_Foo1` is dependent on type `_Name` by field `wrap` ' + r'with `wrap-only` mode enabled'): await client.query(r''' mod_type('_Foo1', 'wpo', false); ''') @@ -1569,6 +1570,8 @@ async def test_assign_in_def(self, client): async def test_export(self, client): script = r''' +try(commit('Source: collection `stuff`')); + new_type('Friend'); new_type('Person'); new_type('Root'); diff --git a/src/ti/collection.c b/src/ti/collection.c index c6b1f41c..491f4814 100644 --- a/src/ti/collection.c +++ b/src/ti/collection.c @@ -69,6 +69,7 @@ ti_collection_t * ti_collection_create( collection->futures = vec_new(4); collection->vtasks = vec_new(4); collection->named_rooms = smap_create(); + collection->commits = NULL; memcpy(&collection->guid, guid, sizeof(guid_t)); diff --git a/src/ti/fn.c b/src/ti/fn.c index 1b38778f..3ef1c737 100644 --- a/src/ti/fn.c +++ b/src/ti/fn.c @@ -14,14 +14,3 @@ int fn_arg_str_slow( name, argn, ti_val_str(val), doc); return e->nr; } - -int fn_commit(const char * name, ti_query_t * query, ex_t * e) -{ - if (ti_query_commits(query) && !query->commit) - ex_set(e, EX_OPERATION, - "function `%s` requires a commit " - "before it can be used in the `%s` scope"DOC_COMMIT, - name, - ti_query_scope_name(query)); - return e->nr; -} \ No newline at end of file From 8d4b6db58992cf49931fc85743e16a928ff380ca Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Tue, 4 Nov 2025 19:12:22 +0100 Subject: [PATCH 10/23] inline --- inc/ti/fn/fn.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/ti/fn/fn.h b/inc/ti/fn/fn.h index eb4f3cfc..4817344a 100644 --- a/inc/ti/fn/fn.h +++ b/inc/ti/fn/fn.h @@ -478,7 +478,7 @@ static inline int fn_not_thingsdb_or_collection_scope( static inline int fn_commit(const char * name, ti_query_t * query, ex_t * e) { - if (ti_query_commits(query) && !query->commit) + if (*ti_query_commits(query) && !query->commit) ex_set(e, EX_OPERATION, "function `%s` requires a commit " "before it can be used in the `%s` scope"DOC_COMMIT, From 50258c00a7446d3054572b6d134aa9c7cd5786f5 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 11:06:26 +0100 Subject: [PATCH 11/23] fixed #424 --- CHANGELOG.md | 3 +- inc/ti/commits.h | 1 + inc/ti/fn/fncommit.h | 11 +-- inc/ti/fn/fntotype.h | 5 +- inc/ti/version.h | 2 +- itest/run_all_tests.py | 2 + itest/test_commits.py | 43 +++++++++++ itest/test_node_functions.py | 4 +- itest/test_relations.py | 4 +- itest/test_thingsdb_functions.py | 4 +- itest/test_type.py | 126 ++++++++++++++++++++++--------- src/ti/commit.c | 11 +-- src/ti/commits.c | 123 ++++++++++++++++++++++++------ src/ti/store/storecommits.c | 5 +- src/ti/type.c | 24 +++--- 15 files changed, 274 insertions(+), 94 deletions(-) create mode 100644 itest/test_commits.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e58297ff..3d3e78e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# v1.8.0-alpha6 +# v1.8.0-alpha8 * Added `TI_PROTO_CLIENT_REQ_EMIT_PEER` protocol, pr #414. * Changed `copy()` behavior for wrapped type, pr #415. @@ -14,6 +14,7 @@ - `history()`: https://docs.thingsdb.io/v1/thingsdb-api/history/ - `set_history()`: https://docs.thingsdb.io/v1/thingsdb-api/set_history/ - `del_history()`: https://docs.thingsdb.io/v1/thingsdb-api/del_history/ +* Fixed handling non-stored thing with `to_type()`, issue #424. # v1.7.6 diff --git a/inc/ti/commits.h b/inc/ti/commits.h index b85b355a..d98438a0 100644 --- a/inc/ti/commits.h +++ b/inc/ti/commits.h @@ -24,6 +24,7 @@ typedef struct ti_vint_t * first; ti_datetime_t * before; ti_datetime_t * after; + ti_vbool_t * has_err; ti_vbool_t * detail; } ti_commits_options_t; diff --git a/inc/ti/fn/fncommit.h b/inc/ti/fn/fncommit.h index a8155261..30ee676f 100644 --- a/inc/ti/fn/fncommit.h +++ b/inc/ti/fn/fncommit.h @@ -3,7 +3,6 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) { const int nargs = fn_get_nargs(nd); - ti_task_t * task; ti_raw_t * message; vec_t ** commits = ti_query_commits(query); ti_commit_t * commit; @@ -81,15 +80,7 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) goto fail0; } - task = ti_task_get_task( - query->change, - query->collection ? query->collection->root : ti.thing0); - - if (!task || ti_task_add_commit_add(task, commit)) - { - ti_commit_destroy(vec_pop(*commits)); /* undo commit */ - ex_set_mem(e); - } + query->commit = commit; fail0: ti_val_unsafe_drop((ti_val_t *) message); diff --git a/inc/ti/fn/fntotype.h b/inc/ti/fn/fntotype.h index fd8b9f58..ded4195e 100644 --- a/inc/ti/fn/fntotype.h +++ b/inc/ti/fn/fntotype.h @@ -12,13 +12,14 @@ static int do__f_to_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) return fn_call_try("to_type", query, nd, e); if (query->collection && - query->collection->root == (ti_thing_t *) query->rval && query->collection->commits && + query->collection->root == (ti_thing_t *) query->rval && !query->commit) { ex_set(e, EX_OPERATION, "function `to_type` requires a commit before " - "it can be used on the `root()` of the `%s` scope"DOC_COMMIT, + "it can be used on the `root()` of a `%s` scope " + "with `history` enabled"DOC_COMMIT, ti_query_scope_name(query)); return e->nr; } diff --git a/inc/ti/version.h b/inc/ti/version.h index 39d28e4a..cbeee5f5 100644 --- a/inc/ti/version.h +++ b/inc/ti/version.h @@ -25,7 +25,7 @@ * "-rc0" * "" */ -#define TI_VERSION_PRE_RELEASE "-alpha7" +#define TI_VERSION_PRE_RELEASE "-alpha8" #define TI_MAINTAINER \ "Jeroen van der Heijden " diff --git a/itest/run_all_tests.py b/itest/run_all_tests.py index aeed45e1..cdfd5122 100755 --- a/itest/run_all_tests.py +++ b/itest/run_all_tests.py @@ -7,6 +7,7 @@ from test_backup import TestBackup from test_changes import TestChanges from test_collection_functions import TestCollectionFunctions +from test_commits import TestCommits from test_datetime import TestDatetime from test_dict import TestDict from test_doc_url import TestDocUrl @@ -78,6 +79,7 @@ def no_mem_test(test_class): run_test(TestBackup(), hide_version=hide_version()) run_test(TestChanges(), hide_version=hide_version()) run_test(TestCollectionFunctions(), hide_version=hide_version()) + run_test(TestCommits(), hide_version=hide_version()) run_test(TestDatetime(), hide_version=hide_version()) run_test(TestDict(), hide_version=hide_version()) if args.doc_test is True: diff --git a/itest/test_commits.py b/itest/test_commits.py new file mode 100644 index 00000000..8ef7bbed --- /dev/null +++ b/itest/test_commits.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +import asyncio +from lib import run_test +from lib import default_test_setup +from lib.testbase import TestBase +from lib.client import get_client +from thingsdb.exceptions import TypeError +from thingsdb.exceptions import NumArgumentsError +from thingsdb.exceptions import LookupError +from thingsdb.exceptions import OperationError + + +class TestCommits(TestBase): + + title = 'Test commit history' + + def with_node1(self): + if hasattr(self, 'node1'): + return True + print(''' + WARNING: Test requires a second node!!! + ''') + + @default_test_setup(num_nodes=1, seed=1, threshold_full_storage=10) + async def async_run(self): + + await self.node0.init_and_run() + + client = await get_client(self.node0) + client.set_default_scope('//stuff') + + # add another node + # if hasattr(self, 'node1'): + # await self.node1.join_until_ready(client) + + await self.run_tests(client) + + client.close() + await client.wait_closed() + + +if __name__ == '__main__': + run_test(TestCommits()) diff --git a/itest/test_node_functions.py b/itest/test_node_functions.py index 11c0bc7e..135dc446 100755 --- a/itest/test_node_functions.py +++ b/itest/test_node_functions.py @@ -84,7 +84,7 @@ async def test_node_info(self, client): node = await client.query('node_info();') - self.assertEqual(len(node), 41) + self.assertEqual(len(node), 42) self.assertIn("node_id", node) self.assertIn("version", node) @@ -127,6 +127,7 @@ async def test_node_info(self, client): self.assertIn('modules_path', node) self.assertIn('architecture', node) self.assertIn('platform', node) + self.assertIn('commit_history', node) self.assertTrue(isinstance(node["node_id"], int)) self.assertTrue(isinstance(node["version"], str)) @@ -169,6 +170,7 @@ async def test_node_info(self, client): self.assertTrue(isinstance(node["modules_path"], str)) self.assertTrue(isinstance(node["architecture"], str)) self.assertTrue(isinstance(node["platform"], str)) + self.assertEqual(node["commit_history"], "disabled") async def test_nodes_info(self, client): with self.assertRaisesRegex( diff --git a/itest/test_relations.py b/itest/test_relations.py index 779f3455..99612add 100755 --- a/itest/test_relations.py +++ b/itest/test_relations.py @@ -29,7 +29,7 @@ async def async_run(self): client = await get_client(self.node0) client.set_default_scope('//stuff') - # add another node otherwise backups are not possible + # add another node if hasattr(self, 'node1'): await self.node1.join_until_ready(client) @@ -415,6 +415,8 @@ async def test_set_set_init_state(self, client0): await client.query(r'''wse();''') res = await client.query(r'''export();''') self.assertEqual(res, r''' +try(commit('Source: collection `stuff`')); + new_type('A'); new_type('B'); new_type('C'); diff --git a/itest/test_thingsdb_functions.py b/itest/test_thingsdb_functions.py index 14f87064..fe06fe39 100755 --- a/itest/test_thingsdb_functions.py +++ b/itest/test_thingsdb_functions.py @@ -118,7 +118,7 @@ async def test_collections_info(self, client): collections = await client.query('collections_info();') self.assertEqual(len(collections), 1) - self.assertEqual(len(collections[0]), 7) + self.assertEqual(len(collections[0]), 8) self.assertIn("collection_id", collections[0]) self.assertIn("next_free_id", collections[0]) @@ -127,6 +127,7 @@ async def test_collections_info(self, client): self.assertIn("created_at", collections[0]) self.assertIn("time_zone", collections[0]) self.assertIn("default_deep", collections[0]) + self.assertIn("commit_history", collections[0]) self.assertTrue(isinstance(collections[0]["collection_id"], int)) self.assertTrue(isinstance(collections[0]["next_free_id"], int)) @@ -135,6 +136,7 @@ async def test_collections_info(self, client): self.assertTrue(isinstance(collections[0]["created_at"], int)) self.assertTrue(isinstance(collections[0]["time_zone"], str)) self.assertTrue(isinstance(collections[0]["default_deep"], int)) + self.assertEqual(collections[0]["commit_history"], "disabled") # at least one info should be checked for a correct created_at info self.assertGreater(collections[0]['created_at'], now - 60) diff --git a/itest/test_type.py b/itest/test_type.py index 5ae4116d..8262cb83 100755 --- a/itest/test_type.py +++ b/itest/test_type.py @@ -37,14 +37,14 @@ async def async_run(self): client.close() await client.wait_closed() - async def test_wrap_only(self, client): + async def _TODO_test_wrap_only(self, client): self.assertFalse(await client.query(r''' try(set_type('X', {arr: '[str?]'}, {1/0})); // tests if type is correctly broken down on error has_type('X'); ''')) - async def test_mod_arr(self, client): + async def _TODO_test_mod_arr(self, client): await client.query(r''' set_type('X', {arr: '[str?]'}); x = X{arr: [nil]}; @@ -52,7 +52,7 @@ async def test_mod_arr(self, client): mod_type('X', 'add', 'arr2', '[str]', [""]); ''') - async def test_set_prop(self, client): + async def _TODO_test_set_prop(self, client): await client.query(r''' set_type('Pet', {}); set_type('setPet', {animal: '{Pet}'}); @@ -80,7 +80,7 @@ async def test_set_prop(self, client): .t.p.animal.add({}); ''') - async def test_method(self, client): + async def _TODO_test_method(self, client): await client.query(r''' set_type('Person', { name: 'str', @@ -136,7 +136,7 @@ async def test_method(self, client): await client.query('''types_info();''') - async def test_new_type(self, client0): + async def _TODO_test_new_type(self, client0): with self.assertRaisesRegex( ValueError, @@ -186,7 +186,7 @@ async def test_new_type(self, client0): client1.close() await client1.wait_closed() - async def test_mod_type_add(self, client0): + async def _TODO_test_mod_type_add(self, client0): await client0.query(r''' set_type(new_type('User'), { name: 'str', @@ -241,7 +241,7 @@ async def test_mod_type_add(self, client0): self.assertIs(iris_node0.get('age'), None) self.assertEqual(iris_node0.get('friend').get('name'), 'Anne') - async def test_enum_wrap_thing(self, client0): + async def _TODO_test_enum_wrap_thing(self, client0): await client0.query(r''' set_type('TColor', { name: 'str', @@ -323,7 +323,7 @@ async def test_enum_wrap_thing(self, client0): self.assertIn('name', color) self.assertEqual(len(color), 2) - async def test_enum_wrap_int(self, client0): + async def _TODO_test_enum_wrap_int(self, client0): await client0.query(r''' set_type('TColor', { name: 'str', @@ -379,7 +379,7 @@ async def test_enum_wrap_int(self, client0): self.assertIn('color', brick) self.assertIsInstance(brick['color'], int) - async def test_wrap_other(self, client0): + async def _TODO_test_wrap_other(self, client0): only_name = await client0.query(r''' set_type('_Name', {name: 'any'}); .only_name = { @@ -415,7 +415,7 @@ async def test_wrap_other(self, client0): client1.close() await client1.wait_closed() - async def test_circular_dep(self, client): + async def _TODO_test_circular_dep(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -472,7 +472,7 @@ async def test_circular_dep(self, client): mod_type('Toe', 'add', 'alt', 'Tic'); ''') - async def test_mod_del(self, client): + async def _TODO_test_mod_del(self, client): await client.query(r''' new_type('Foo'); set_type('Foo', { @@ -491,7 +491,7 @@ async def test_mod_del(self, client): mod_type('Foo', 'del', 'foo'); ''') - async def test_mod(self, client): + async def _TODO_test_mod(self, client): await client.query(r''' set_type('Person', { name: 'str', @@ -860,7 +860,7 @@ async def test_mod(self, client): r'type `Person` have been found'): await client.query(r'mod_type("Person", "wpo", true);') - async def test_mod_type(self, client): + async def _TODO_test_mod_type(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -910,7 +910,7 @@ async def test_mod_type(self, client): del_type('Bar'); ''') - async def test_type_mismatch(self, client): + async def _TODO_test_type_mismatch(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -947,7 +947,7 @@ async def test_type_mismatch(self, client): }); ''') - async def test_migrate(self, client0): + async def _TODO_test_migrate(self, client0): client1 = await get_client(self.node1) client1.set_default_scope('//stuff') @@ -978,7 +978,7 @@ async def test_migrate(self, client0): client1 = await get_client(self.node1) client1.set_default_scope('//stuff') - async def test_sync_add(self, client0): + async def _TODO_test_sync_add(self, client0): await client0.query(r''' set_type('Book', { title: 'str', @@ -1051,7 +1051,7 @@ async def test_sync_add(self, client0): self.assertEqual(books0, {"own": [], "fav": None, "unique": []}) self.assertEqual(books1, {"own": [], "fav": None, "unique": []}) - async def test_circular_del(self, client): + async def _TODO_test_circular_del(self, client): await client.query(r''' new_type('Foo'); new_type('Bar'); @@ -1076,7 +1076,7 @@ async def test_circular_del(self, client): ''') self.assertIs(res, None) - async def test_del_type(self, client): + async def _TODO_test_del_type(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -1159,7 +1159,7 @@ async def test_del_type(self, client): del_type('Toe'); ''') - async def test_type_definitions(self, client): + async def _TODO_test_type_definitions(self, client): await client.query(r''' set_type('_Str', {test: 'str'}); set_type('_Utf8', {test: 'utf8'}); @@ -1337,7 +1337,7 @@ async def test_type_definitions(self, client): await client.query(r'_Nint{test:-6}.wrap("_Number")'), {'test': -6}) - async def test_mod_type_add_closure_and_ts(self, client0): + async def _TODO_test_mod_type_add_closure_and_ts(self, client0): now = int(time.time()) await client0.query(r''' @@ -1407,7 +1407,7 @@ async def test_mod_type_add_closure_and_ts(self, client0): client1.close() await client1.wait_closed() - async def test_mod_type_mod_closure(self, client0): + async def _TODO_test_mod_type_mod_closure(self, client0): await client0.query(r''' set_type('Chat', { name: 'str', @@ -1475,7 +1475,7 @@ async def test_mod_type_mod_closure(self, client0): client1.close() await client1.wait_closed() - async def test_mod_type_mod_advanced0(self, client0): + async def _TODO_test_mod_type_mod_advanced0(self, client0): with self.assertRaisesRegex( OperationError, r'field `chat` is added to type `Room` but at least one ' @@ -1511,7 +1511,7 @@ async def test_mod_type_mod_advanced0(self, client0): client1.close() await client1.wait_closed() - async def test_mod_type_mod_advanced1(self, client0): + async def _TODO_test_mod_type_mod_advanced1(self, client0): await client0.query(r''' set_type('Chat', { messages: '[str]' @@ -1555,7 +1555,7 @@ async def test_mod_type_mod_advanced1(self, client0): client1.close() await client1.wait_closed() - async def test_mod_type_mod_advanced2(self, client0): + async def _TODO_test_mod_type_mod_advanced2(self, client0): with self.assertRaisesRegex( OperationError, r'field `chat` on type `Room` is modified but at least one ' @@ -1603,7 +1603,7 @@ async def test_mod_type_mod_advanced2(self, client0): client1.close() await client1.wait_closed() - async def test_wpo(self, client0): + async def _TODO_test_wpo(self, client0): await client0.query(r''' new_type('A', true); new_type('B', false); @@ -1641,7 +1641,7 @@ async def test_wpo(self, client0): client1.close() await client1.wait_closed() - async def test_err_handling(self, client0): + async def _TODO_test_err_handling(self, client0): with self.assertRaisesRegex( TypeError, r'invalid declaration for `x` on type `A`; ' @@ -1667,7 +1667,7 @@ async def test_err_handling(self, client0): client1.close() await client1.wait_closed() - async def test_init_create(self, client0): + async def _TODO_test_init_create(self, client0): await client0.query(r''' set_type('X', { a: new_type('A'), @@ -1686,7 +1686,7 @@ async def test_init_create(self, client0): client1.close() await client1.wait_closed() - async def test_rename(self, client0): + async def _TODO_test_rename(self, client0): await client0.query(r''' set_type('A', {i: 'int'}); set_type('B', {a: 'A'}); @@ -1742,7 +1742,7 @@ async def test_rename(self, client0): client1.close() await client1.wait_closed() - async def test_rename_prop(self, client0): + async def _TODO_test_rename_prop(self, client0): await client0.query(r''' set_type('Test', { arr: '[]', @@ -1795,7 +1795,7 @@ async def test_rename_prop(self, client0): client1.close() await client1.wait_closed() - async def test_wrap(self, client0): + async def _TODO_test_wrap(self, client0): await client0.query(r''' new_type('_A'); // true through mod_type new_type('_B'); // true through set_type @@ -1836,7 +1836,7 @@ async def test_wrap(self, client0): client1.close() await client1.wait_closed() - async def test_same_deep(self, client): + async def _TODO_test_same_deep(self, client): res = await client.query(r"""//ti new_type('User'); set_type('User', { @@ -1901,7 +1901,7 @@ async def test_same_deep(self, client): "nosamedeep": {}, }) - async def test_type_name_id(self, client): + async def _TODO_test_type_name_id(self, client): with self.assertRaisesRegex( LookupError, r'multiple Id \(\'#\'\) definitions on type `User`'): @@ -2024,7 +2024,7 @@ async def test_type_name_id(self, client): client1.close() await client1.wait_closed() - async def test_prefix_flags(self, client): + async def _TODO_test_prefix_flags(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `name` on type `User`; ' @@ -2112,7 +2112,7 @@ async def test_prefix_flags(self, client): }, }) - async def test_rename_type_with_flags(self, client): + async def _TODO_test_rename_type_with_flags(self, client): res = await client.query(r"""//ti new_type('A'); set_type('B', { @@ -2136,7 +2136,7 @@ async def test_rename_type_with_flags(self, client): [['a0', 'AA'], ['a1', 'AA?'], ['a2', '^AA'], ['a3', '!?&+^[AA?]?']] ]) - async def test_email(self, client): + async def _TODO_test_email(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `def` on type `E`; ' @@ -2185,7 +2185,7 @@ async def test_email(self, client): E{}.email = 'abc'; """) - async def test_url(self, client): + async def _TODO_test_url(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `def` on type `E`; ' @@ -2234,7 +2234,7 @@ async def test_url(self, client): E{}.url = 'abc'; """) - async def test_tel(self, client): + async def _TODO_test_tel(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `def` on type `E`; ' @@ -2283,6 +2283,58 @@ async def test_tel(self, client): E{}.tel = 'abc'; """) + async def test_to_type_no_store(self, client0): + # this test is for bug #424 + q0 = client0.query + await q0("""//ti- + set_type('T', {name: 'str<::default>'}); + t = {}; + t.to_type('T'); + t; + .o = {}; + .x = 42; + """) + + client1 = await get_client(self.node1) + client1.set_default_scope('//stuff') + + await self.wait_nodes_ready(client0) + q1 = client1.query + + for q in (q0, q1): + r = await q("""//ti + .x; + """) + self.assertEqual(r, 42) + + r = await q("""//ti + nse(); + t = {}; + t.to_type('T'); + change_id(); + """) + self.assertIs(r, None) + + with self.assertRaisesRegex( + OperationError, + r'operation on a stored thing; remove `nse\(...\)` to ' + r'enforce a change'): + await q0("""//ti + nse(); + .o.to_type('T'); + """) + + await q0("""//ti + .o.to_type('T'); + """) + + for q in (q0, q1): + r = await q("""//ti + wse(); + .o.name; + """) + self.assertEqual(r, "default") + if __name__ == '__main__': run_test(TestType()) diff --git a/src/ti/commit.c b/src/ti/commit.c index 62f1106a..13f700cb 100644 --- a/src/ti/commit.c +++ b/src/ti/commit.c @@ -63,13 +63,14 @@ static ti_commit_t * commit_create( ti_commit_t * ti_commit_from_up(mp_unp_t * up) { mp_obj_t obj, mp_id, mp_ts, mp_code, mp_by, mp_message, mp_err_msg; + if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 6 || mp_next(up, &mp_id) != MP_U64 || mp_next(up, &mp_ts) != MP_U64 || mp_next(up, &mp_code) != MP_STR || - mp_next(up, &mp_by) != MP_STR || mp_next(up, &mp_message) != MP_STR || - (mp_next(up, &mp_err_msg) != MP_STR && mp_err_msg.tp == MP_NIL)) + mp_next(up, &mp_by) != MP_STR || + (mp_next(up, &mp_err_msg) != MP_STR && mp_err_msg.tp != MP_NIL)) return NULL; return commit_create( mp_id.via.u64, @@ -154,16 +155,16 @@ int ti_commit_to_client_pk( mp_pack_strn(pk, commit->by->data, commit->by->n) || mp_pack_str(pk, "message") || - mp_pack_strn(pk, commit->message, commit->message->n) || + mp_pack_strn(pk, commit->message->data, commit->message->n) || (detail && ( mp_pack_str(pk, "code") || - mp_pack_strn(pk, commit->code, commit->code->n) + mp_pack_strn(pk, commit->code->data, commit->code->n) )) || (commit->err_msg && ( mp_pack_str(pk, "err_msg") || - mp_pack_strn(pk, commit->err_msg, commit->err_msg->n) + mp_pack_strn(pk, commit->err_msg->data, commit->err_msg->n) )) ); } diff --git a/src/ti/commits.c b/src/ti/commits.c index 71c051d3..f4ef2847 100644 --- a/src/ti/commits.c +++ b/src/ti/commits.c @@ -20,6 +20,8 @@ static int commits__options( { if (ti_raw_eq_strn(key, "scope", strlen("scope"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_str(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -33,6 +35,8 @@ static int commits__options( } if (ti_raw_eq_strn(key, "contains", strlen("contains"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_str(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -46,6 +50,8 @@ static int commits__options( } if (ti_raw_eq_strn(key, "match", strlen("match"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_regex(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -59,6 +65,8 @@ static int commits__options( } if (ti_raw_eq_strn(key, "id", strlen("id"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_int(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -72,6 +80,8 @@ static int commits__options( } if (ti_raw_eq_strn(key, "last", strlen("last"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_int(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -85,6 +95,8 @@ static int commits__options( } if (ti_raw_eq_strn(key, "first", strlen("first"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_int(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -98,6 +110,8 @@ static int commits__options( } if (ti_raw_eq_strn(key, "before", strlen("before"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_datetime(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -112,6 +126,8 @@ static int commits__options( } if (ti_raw_eq_strn(key, "after", strlen("after"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_datetime(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -124,8 +140,25 @@ static int commits__options( w->options->after = (ti_datetime_t *) val; return 0; } + if (ti_raw_eq_strn(key, "has_err", strlen("has_err"))) + { + if (ti_val_is_nil(val)) + return 0; + if (!ti_val_is_bool(val)) + { + ex_set(w->e, EX_TYPE_ERROR, + "expecting `has_err` to be of type `"TI_VAL_BOOL_S"` but " + "got type `%s` instead", + ti_val_str(val)); + return w->e->nr; + } + w->options->has_err = (ti_vbool_t *) val; + return 0; + } if (w->allow_detail && ti_raw_eq_strn(key, "detail", strlen("detail"))) { + if (ti_val_is_nil(val)) + return 0; if (!ti_val_is_bool(val)) { ex_set(w->e, EX_TYPE_ERROR, @@ -260,23 +293,25 @@ vec_t ** ti_commits_from_scope(ti_raw_t * scope, ex_t * e) switch(scope_.tp) { - case TI_SCOPE_THINGSDB: - return &ti.commits; - case TI_SCOPE_NODE: - return NULL; - case TI_SCOPE_COLLECTION: - { - ti_collection_t * collection = ti_collections_get_by_strn( - scope_.via.collection_name.name, - scope_.via.collection_name.sz); - if (collection) - return &collection->commits; + case TI_SCOPE_THINGSDB: + return &ti.commits; + case TI_SCOPE_NODE: + ex_set(e, EX_LOOKUP_ERROR, "no commit history in node scope"); + return NULL; + case TI_SCOPE_COLLECTION: + { + ti_collection_t * collection = ti_collections_get_by_strn( + scope_.via.collection_name.name, + scope_.via.collection_name.sz); + if (collection) + return &collection->commits; - ex_set(e, EX_LOOKUP_ERROR, "collection `%.*s` not found", - scope_.via.collection_name.sz, - scope_.via.collection_name.name); - } + ex_set(e, EX_LOOKUP_ERROR, "collection `%.*s` not found", + scope_.via.collection_name.sz, + scope_.via.collection_name.name); + } } + ex_set_internal(e); return NULL; } @@ -293,7 +328,7 @@ int ti_commits_set_history(vec_t ** commits, _Bool state) return 0; *commits = vec_new(8); - return !!(*commits); + return -!(*commits); } ti_commits_destroy(commits); return 0; @@ -321,20 +356,63 @@ static _Bool commits__match( ) )) return false; + if (options->id && commit->id != (uint64_t) VINT(options->id)) return false; return true; } +static void commits__init( + vec_t * commits, + ti_commits_options_t * options, + uint32_t * begin, + uint32_t * end) +{ + uint32_t n; + *begin = 0; + *end = commits->n; + + if (options->before) + { + uint32_t i = 0; + for (vec_each(commits, ti_commit_t, commit), ++i) + if (commit->ts >= options->before->ts) + break; + *end = i; + } + if (options->after) + { + uint32_t i = *end - 1; + for (vec_each_rev(commits, ti_commit_t, commit) --i) + if (commit->ts <= options->after->ts) + break; + *begin = i; + } + + n = *end > *begin ? *end - *begin : 0; + + if (options->last && n > options->last->int_) + *begin = *end - options->last->int_; + + if (options->first && n > options->first->int_) + *end = *begin + options->first->int_; +} + vec_t * ti_commits_find(vec_t * commits, ti_commits_options_t * options) { + uint32_t begin, end;; vec_t * vec = vec_new(8); if (!vec) return NULL; - for (vec_each(commits, ti_commit_t, commit)) + + commits__init(commits, options, &begin, &end); + for (;begin < end; ++begin) + { + ti_commit_t * commit = VEC_get(commits, begin); if (commits__match(commit, options) && vec_push(&vec, commit)) goto fail; + } return vec; fail: @@ -344,16 +422,19 @@ vec_t * ti_commits_find(vec_t * commits, ti_commits_options_t * options) vec_t * ti_commits_del(vec_t ** commits, ti_commits_options_t * options) { + uint32_t begin, end; vec_t * vec = vec_new(8); - uint32_t n = (*commits)->n; if (!vec) return NULL; - for (vec_each_rev(*commits, ti_commit_t, commit) --n) + commits__init(*commits, options, &begin, &end); + while (begin <= --end) + { + ti_commit_t * commit = VEC_get(*commits, end); if (commits__match(commit, options) && - vec_push(&vec, vec_remove(*commits, n))) + vec_push(&vec, vec_remove(*commits, end))) goto fail; - + } (void) vec_shrink(commits); return vec; fail: diff --git a/src/ti/store/storecommits.c b/src/ti/store/storecommits.c index c7e1006f..8caeac4a 100644 --- a/src/ti/store/storecommits.c +++ b/src/ti/store/storecommits.c @@ -87,7 +87,6 @@ int ti_store_commits_restore(vec_t ** commits, const char * fn) goto fail0; mp_unp_init(&up, fmap.data, fmap.n); - if (mp_next(&up, &obj) != MP_MAP || obj.via.sz > 1) goto fail1; @@ -97,7 +96,9 @@ int ti_store_commits_restore(vec_t ** commits, const char * fn) goto done; /* done, commits disabled */ } - if (ti_commits_set_history(commits, true)) + if (mp_skip(&up) != MP_STR || /* commits */ + mp_next(&up, &obj) != MP_ARR || + ti_commits_set_history(commits, true)) goto fail1; for (i = obj.via.sz; i--;) diff --git a/src/ti/type.c b/src/ti/type.c index fed4615b..f974f4d3 100644 --- a/src/ti/type.c +++ b/src/ti/type.c @@ -1404,12 +1404,20 @@ int ti_type_convert( ti_val_t ** val = (ti_val_t **) vec_get_addr(w.vec, field->idx); if (!*val) { - if (change) + /* Depending from where called, we might have a thing->id and no + * change (when processing a change from disk or other node). + * We also might have a change, and no thing->id. see bug #424 */ + *val = field->dval_cb(field); + if (!val) + { + ex_set_mem(e); + goto fail0; + } + ti_val_attach(*val, thing, field); + if (change && thing->id) { ti_task_t * task = ti_task_get_task(change, thing); - - *val = field->dval_cb(field); - if (!*val || !task || ti_task_add_set( + if (!task || ti_task_add_set( task, (ti_raw_t *) field->name, *val)) @@ -1417,14 +1425,6 @@ int ti_type_convert( ex_set_mem(e); goto fail0; } - ti_val_attach(*val, thing, field); - } - else - { - ex_set(e, EX_TYPE_ERROR, - "conversion failed; property `%s` is missing", - field->name->str); - goto fail0; } } } From d63aeeccbaf8bbb9e7e5039d31da6e62dc2df1ec Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 12:43:10 +0100 Subject: [PATCH 12/23] work on commits testing --- inc/ti/commits.h | 3 + inc/ti/fn/fndelhistory.h | 2 +- inc/ti/fn/fnhistory.h | 2 +- inc/ti/fn/fnsethistory.h | 25 ++-- inc/ti/fn/fntotype.h | 2 +- itest/test_commits.py | 265 +++++++++++++++++++++++++++++++++++++-- itest/test_type.py | 79 ++++++------ src/ti/commits.c | 30 +++-- 8 files changed, 334 insertions(+), 74 deletions(-) mode change 100644 => 100755 itest/test_commits.py diff --git a/inc/ti/commits.h b/inc/ti/commits.h index d98438a0..f43e78ea 100644 --- a/inc/ti/commits.h +++ b/inc/ti/commits.h @@ -4,6 +4,7 @@ #ifndef TI_COMMITS_H_ #define TI_COMMITS_H_ +#include #include #include #include @@ -14,6 +15,8 @@ #include #include +#define TI_COMMITS_MASK TI_AUTH_QUERY|TI_AUTH_CHANGE + typedef struct { ti_raw_t * scope; diff --git a/inc/ti/fn/fndelhistory.h b/inc/ti/fn/fndelhistory.h index e3e4bb87..fe8ba8ea 100644 --- a/inc/ti/fn/fndelhistory.h +++ b/inc/ti/fn/fndelhistory.h @@ -16,7 +16,7 @@ static int do__f_del_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) ti_commits_history(&history, &options, e)) return e->nr; - if (ti_access_check_err(*history.access, query->user, TI_AUTH_GRANT, e)) + if (ti_access_check_err(*history.access, query->user, TI_COMMITS_MASK, e)) return e->nr; deleted = ti_commits_del(history.commits, &options); diff --git a/inc/ti/fn/fnhistory.h b/inc/ti/fn/fnhistory.h index 6c98e725..25e484b3 100644 --- a/inc/ti/fn/fnhistory.h +++ b/inc/ti/fn/fnhistory.h @@ -17,7 +17,7 @@ static int do__f_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) ti_commits_history(&history, &options, e)) return e->nr; - if (ti_access_check_err(*history.access, query->user, TI_AUTH_GRANT, e)) + if (ti_access_check_err(*history.access, query->user, TI_COMMITS_MASK, e)) return e->nr; filtered = ti_commits_find(*history.commits, &options); diff --git a/inc/ti/fn/fnsethistory.h b/inc/ti/fn/fnsethistory.h index 8ff1564d..b8e34202 100644 --- a/inc/ti/fn/fnsethistory.h +++ b/inc/ti/fn/fnsethistory.h @@ -14,7 +14,7 @@ static int do__f_set_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) return e->nr; access_ = ti_val_get_access(query->rval, e, &scope_id); - if (e->nr || ti_access_check_err(*access_, query->user, TI_AUTH_GRANT, e)) + if (e->nr || ti_access_check_err(*access_, query->user, TI_COMMITS_MASK, e)) return e->nr; commits = ti_commits_from_scope((ti_raw_t *) query->rval, e); @@ -33,18 +33,21 @@ static int do__f_set_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) ti_val_unsafe_drop(query->rval); query->rval = (ti_val_t *) ti_nil_get(); - if (ti_commits_set_history(commits, state)) + if (state != !!(*commits)) { - ex_set_mem(e); - return e->nr; + if (ti_commits_set_history(commits, state)) + { + ex_set_mem(e); + return e->nr; + } + + task = ti_task_get_task( + query->change, + query->collection ? query->collection->root : ti.thing0); + + if (!task || ti_task_add_set_history(task, scope_id, state)) + ex_set_mem(e); /* task cleanup is not required */ } - task = ti_task_get_task( - query->change, - query->collection ? query->collection->root : ti.thing0); - - if (!task || ti_task_add_set_history(task, scope_id, state)) - ex_set_mem(e); /* task cleanup is not required */ - return e->nr; } diff --git a/inc/ti/fn/fntotype.h b/inc/ti/fn/fntotype.h index ded4195e..b2e3baa6 100644 --- a/inc/ti/fn/fntotype.h +++ b/inc/ti/fn/fntotype.h @@ -19,7 +19,7 @@ static int do__f_to_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) ex_set(e, EX_OPERATION, "function `to_type` requires a commit before " "it can be used on the `root()` of a `%s` scope " - "with `history` enabled"DOC_COMMIT, + "with commit history enabled"DOC_COMMIT, ti_query_scope_name(query)); return e->nr; } diff --git a/itest/test_commits.py b/itest/test_commits.py old mode 100644 new mode 100755 index 8ef7bbed..2cda52e4 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -8,6 +8,7 @@ from thingsdb.exceptions import NumArgumentsError from thingsdb.exceptions import LookupError from thingsdb.exceptions import OperationError +from thingsdb.exceptions import ForbiddenError class TestCommits(TestBase): @@ -26,17 +27,267 @@ async def async_run(self): await self.node0.init_and_run() - client = await get_client(self.node0) - client.set_default_scope('//stuff') + client0 = await get_client(self.node0) + client0.set_default_scope('//stuff') # add another node - # if hasattr(self, 'node1'): - # await self.node1.join_until_ready(client) + if hasattr(self, 'node1'): + await self.node1.join_until_ready(client0) + + client1 = await get_client(self.node0) + client1.set_default_scope('//stuff') + + await self.run_tests(client0.query, client1.query) + + for client in (client0, client1): + client.close() + await client.wait_closed() + + async def test_set_history(self, q0, q1): + with self.assertRaisesRegex( + LookupError, + r'function `set_history` is undefined in the `@node` scope; ' + r'you might want to query the `@thingsdb` scope\?'): + await q0("""//ti + set_history(); + """, scope='/n') + + with self.assertRaisesRegex( + NumArgumentsError, + r'function `set_history` takes 2 arguments but 0 were given;'): + await q0("""//ti + set_history(); + """, scope='/t') + + with self.assertRaisesRegex( + TypeError, + r'expecting a scope to be of type `str` but ' + r'got type `nil` instead'): + await q0("""//ti + set_history(nil, true); + """, scope='/t') + + with self.assertRaisesRegex( + ValueError, + r'invalid scope; scopes must not be empty'): + await q0("""//ti + set_history("", true); + """, scope='/t') + + with self.assertRaisesRegex( + OperationError, + r'commit history is not supported by `@node` scopes'): + await q0("""//ti + set_history("/n", true); + """, scope='/t') + + with self.assertRaisesRegex( + LookupError, + r'collection `s` not found'): + await q0("""//ti + set_history("//s", true); + """, scope='/t') + + with self.assertRaisesRegex( + TypeError, + r'function `set_history` expects argument 2 to be of ' + r'type `bool` but got type `int` instead;'): + await q0("""//ti + set_history("//stuff", 0); + """, scope='/t') + + await q0("""//ti + set_history("//stuff", true); + """, scope='/t') + for q in (q0, q1): + r = await q("""//ti + wse(); + collection_info('stuff').load().commit_history; + """, scope='/t') + self.assertEqual(r, 0) + + await q0("""//ti + set_history("//stuff", false); + """, scope='/t') + for q in (q0, q1): + r = await q("""//ti + wse(); + collection_info('stuff').load().commit_history; + """, scope='/t') + self.assertEqual(r, 'disabled') + + await q0("""//ti + set_history("/t", true); + """, scope='/t') + for q in (q0, q1): + r = await q("""//ti + wse(); + node_info().load().commit_history; + """, scope='/n') + self.assertEqual(r, 0) + + await q0("""//ti + set_history("/t", false); + """, scope='/t') + for q in (q0, q1): + r = await q("""//ti + wse(); + node_info().load().commit_history; + """, scope='/n') + self.assertEqual(r, 'disabled') + + async def test_access(self, q0, q1): + await q0("""//ti + new_collection('junk'); + new_user('test1'); + set_password('test1', 'test'); + grant('/t', 'test1', QUERY); + grant('//stuff', 'test1', QUERY|CHANGE); + grant('//junk', 'test1', QUERY); + """, scope='/t') + + q = (await get_client( + self.node0, + auth=['test1', 'test'], + auto_reconnect=False)).query + + with self.assertRaisesRegex( + ForbiddenError, + r'user `test1` is missing the required ' + r'privileges \(`CHANGE`\) on scope `@thingsdb`;'): + await q("""//ti + set_history('//junk', true); + """, scope='/t') + + await q0("""//ti + grant('/t', 'test1', QUERY|CHANGE); + """, scope='/t') + + with self.assertRaisesRegex( + ForbiddenError, + r'user `test1` is missing the required ' + r'privileges \(`QUERY\|CHANGE`\) on ' + r'scope `@collection:junk`;'): + await q("""//ti + set_history('//junk', true); + """, scope='/t') + await q("""//ti + set_history('//stuff', true); + set_history('/t', false); + """, scope='/t') + + async def test_history(self, q0, q1): + await q0("""//ti + set_history('//stuff', false); + set_history('/t', false); + """, scope='/t') + + with self.assertRaisesRegex( + LookupError, + r'function `history` is undefined in the `@node` scope; ' + r'you might want to query the `@thingsdb` scope\?'): + await q0("""//ti + history({last:0}); + """, scope='/n') + + with self.assertRaisesRegex( + LookupError, + r'no scope found with commit history enabled;'): + await q0("""//ti + history({last:0}); + """, scope='/t') + + with self.assertRaisesRegex( + OperationError, + r'commit history is not enabled for the `/t` scope'): + await q0("""//ti + history({scope: '/t', last:0}); + """, scope='/t') + + with self.assertRaisesRegex( + OperationError, + r'commit history is not enabled for the `//stuff` scope'): + await q0("""//ti + history({scope: '//stuff', last:0}); + """, scope='/t') + + with self.assertRaisesRegex( + OperationError, + r'commit history is not supported by `@node` scopes'): + await q0("""//ti + history({scope: '/n', last:0}); + """, scope='/t') + + with self.assertRaisesRegex( + LookupError, + r'xxx'): + await q0("""//ti + history({scope: '//s', last:0}); + """, scope='/t') + + await q0("""//ti + set_history('//stuff', true); + set_history('/t', true); + """, scope='/t') + + with self.assertRaisesRegex( + LookupError, + r'more than one scope has commit history enabled; you must ' + r'use the `scope` option to tell ThingsDB which scope ' + r'to use;'): + await q0("""//ti + history({last:0}); + """, scope='/t') + + with self.assertRaisesRegex( + LookupError, + r'at least one of the following options must be set: ' + r'`contains`, `match`, `id`, `first`, `last`, `before`, ' + r'`after`, `has_err`;'): + await q0("""//ti + history({scope: '//stuff'}); + """, scope='/t') + + async def test_commit(self, q0, q1): + await q0("""//ti + set_history('//stuff', false); + set_history('/t', false); + """, scope='/t') + with self.assertRaisesRegex( + LookupError, + r'xxxx'): + await q0("""//ti + commit('Test'); + """, scope='/n') + with self.assertRaisesRegex( + LookupError, + r'xxxx'): + await q0("""//ti + commit('Test'); + new_type('A'); + """) + with self.assertRaisesRegex( + NumArgumentsError, + r'xxxx'): + await q0("""//ti + commit(); + new_type('A'); + """) + with self.assertRaisesRegex( + NumArgumentsError, + r'xxxx'): + await q0("""//ti + commit(123); + new_type('A'); + """) + with self.assertRaisesRegex( + NumArgumentsError, + r'xxxx'): + await q0("""//ti + commit('Test'); + """) - await self.run_tests(client) - client.close() - await client.wait_closed() if __name__ == '__main__': diff --git a/itest/test_type.py b/itest/test_type.py index 8262cb83..3042a010 100755 --- a/itest/test_type.py +++ b/itest/test_type.py @@ -1,19 +1,14 @@ #!/usr/bin/env python import asyncio -import pickle import time from lib import run_test from lib import default_test_setup from lib.testbase import TestBase from lib.client import get_client -from thingsdb.exceptions import AssertionError from thingsdb.exceptions import ValueError from thingsdb.exceptions import TypeError from thingsdb.exceptions import NumArgumentsError -from thingsdb.exceptions import BadDataError from thingsdb.exceptions import LookupError -from thingsdb.exceptions import OverflowError -from thingsdb.exceptions import ZeroDivisionError from thingsdb.exceptions import OperationError @@ -37,14 +32,14 @@ async def async_run(self): client.close() await client.wait_closed() - async def _TODO_test_wrap_only(self, client): + async def test_wrap_only(self, client): self.assertFalse(await client.query(r''' try(set_type('X', {arr: '[str?]'}, {1/0})); // tests if type is correctly broken down on error has_type('X'); ''')) - async def _TODO_test_mod_arr(self, client): + async def test_mod_arr(self, client): await client.query(r''' set_type('X', {arr: '[str?]'}); x = X{arr: [nil]}; @@ -52,7 +47,7 @@ async def _TODO_test_mod_arr(self, client): mod_type('X', 'add', 'arr2', '[str]', [""]); ''') - async def _TODO_test_set_prop(self, client): + async def test_set_prop(self, client): await client.query(r''' set_type('Pet', {}); set_type('setPet', {animal: '{Pet}'}); @@ -80,7 +75,7 @@ async def _TODO_test_set_prop(self, client): .t.p.animal.add({}); ''') - async def _TODO_test_method(self, client): + async def test_method(self, client): await client.query(r''' set_type('Person', { name: 'str', @@ -136,7 +131,7 @@ async def _TODO_test_method(self, client): await client.query('''types_info();''') - async def _TODO_test_new_type(self, client0): + async def test_new_type(self, client0): with self.assertRaisesRegex( ValueError, @@ -186,7 +181,7 @@ async def _TODO_test_new_type(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_mod_type_add(self, client0): + async def test_mod_type_add(self, client0): await client0.query(r''' set_type(new_type('User'), { name: 'str', @@ -241,7 +236,7 @@ async def _TODO_test_mod_type_add(self, client0): self.assertIs(iris_node0.get('age'), None) self.assertEqual(iris_node0.get('friend').get('name'), 'Anne') - async def _TODO_test_enum_wrap_thing(self, client0): + async def test_enum_wrap_thing(self, client0): await client0.query(r''' set_type('TColor', { name: 'str', @@ -323,7 +318,7 @@ async def _TODO_test_enum_wrap_thing(self, client0): self.assertIn('name', color) self.assertEqual(len(color), 2) - async def _TODO_test_enum_wrap_int(self, client0): + async def test_enum_wrap_int(self, client0): await client0.query(r''' set_type('TColor', { name: 'str', @@ -379,7 +374,7 @@ async def _TODO_test_enum_wrap_int(self, client0): self.assertIn('color', brick) self.assertIsInstance(brick['color'], int) - async def _TODO_test_wrap_other(self, client0): + async def test_wrap_other(self, client0): only_name = await client0.query(r''' set_type('_Name', {name: 'any'}); .only_name = { @@ -415,7 +410,7 @@ async def _TODO_test_wrap_other(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_circular_dep(self, client): + async def test_circular_dep(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -472,7 +467,7 @@ async def _TODO_test_circular_dep(self, client): mod_type('Toe', 'add', 'alt', 'Tic'); ''') - async def _TODO_test_mod_del(self, client): + async def test_mod_del(self, client): await client.query(r''' new_type('Foo'); set_type('Foo', { @@ -491,7 +486,7 @@ async def _TODO_test_mod_del(self, client): mod_type('Foo', 'del', 'foo'); ''') - async def _TODO_test_mod(self, client): + async def test_mod(self, client): await client.query(r''' set_type('Person', { name: 'str', @@ -860,7 +855,7 @@ async def _TODO_test_mod(self, client): r'type `Person` have been found'): await client.query(r'mod_type("Person", "wpo", true);') - async def _TODO_test_mod_type(self, client): + async def test_mod_type(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -910,7 +905,7 @@ async def _TODO_test_mod_type(self, client): del_type('Bar'); ''') - async def _TODO_test_type_mismatch(self, client): + async def test_type_mismatch(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -947,7 +942,7 @@ async def _TODO_test_type_mismatch(self, client): }); ''') - async def _TODO_test_migrate(self, client0): + async def test_migrate(self, client0): client1 = await get_client(self.node1) client1.set_default_scope('//stuff') @@ -978,7 +973,7 @@ async def _TODO_test_migrate(self, client0): client1 = await get_client(self.node1) client1.set_default_scope('//stuff') - async def _TODO_test_sync_add(self, client0): + async def test_sync_add(self, client0): await client0.query(r''' set_type('Book', { title: 'str', @@ -1051,7 +1046,7 @@ async def _TODO_test_sync_add(self, client0): self.assertEqual(books0, {"own": [], "fav": None, "unique": []}) self.assertEqual(books1, {"own": [], "fav": None, "unique": []}) - async def _TODO_test_circular_del(self, client): + async def test_circular_del(self, client): await client.query(r''' new_type('Foo'); new_type('Bar'); @@ -1076,7 +1071,7 @@ async def _TODO_test_circular_del(self, client): ''') self.assertIs(res, None) - async def _TODO_test_del_type(self, client): + async def test_del_type(self, client): await client.query(r''' new_type('Tic'); new_type('Tac'); @@ -1159,7 +1154,7 @@ async def _TODO_test_del_type(self, client): del_type('Toe'); ''') - async def _TODO_test_type_definitions(self, client): + async def test_type_definitions(self, client): await client.query(r''' set_type('_Str', {test: 'str'}); set_type('_Utf8', {test: 'utf8'}); @@ -1337,7 +1332,7 @@ async def _TODO_test_type_definitions(self, client): await client.query(r'_Nint{test:-6}.wrap("_Number")'), {'test': -6}) - async def _TODO_test_mod_type_add_closure_and_ts(self, client0): + async def test_mod_type_add_closure_and_ts(self, client0): now = int(time.time()) await client0.query(r''' @@ -1407,7 +1402,7 @@ async def _TODO_test_mod_type_add_closure_and_ts(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_mod_type_mod_closure(self, client0): + async def test_mod_type_mod_closure(self, client0): await client0.query(r''' set_type('Chat', { name: 'str', @@ -1475,7 +1470,7 @@ async def _TODO_test_mod_type_mod_closure(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_mod_type_mod_advanced0(self, client0): + async def test_mod_type_mod_advanced0(self, client0): with self.assertRaisesRegex( OperationError, r'field `chat` is added to type `Room` but at least one ' @@ -1511,7 +1506,7 @@ async def _TODO_test_mod_type_mod_advanced0(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_mod_type_mod_advanced1(self, client0): + async def test_mod_type_mod_advanced1(self, client0): await client0.query(r''' set_type('Chat', { messages: '[str]' @@ -1555,7 +1550,7 @@ async def _TODO_test_mod_type_mod_advanced1(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_mod_type_mod_advanced2(self, client0): + async def test_mod_type_mod_advanced2(self, client0): with self.assertRaisesRegex( OperationError, r'field `chat` on type `Room` is modified but at least one ' @@ -1603,7 +1598,7 @@ async def _TODO_test_mod_type_mod_advanced2(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_wpo(self, client0): + async def test_wpo(self, client0): await client0.query(r''' new_type('A', true); new_type('B', false); @@ -1641,7 +1636,7 @@ async def _TODO_test_wpo(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_err_handling(self, client0): + async def test_err_handling(self, client0): with self.assertRaisesRegex( TypeError, r'invalid declaration for `x` on type `A`; ' @@ -1667,7 +1662,7 @@ async def _TODO_test_err_handling(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_init_create(self, client0): + async def test_init_create(self, client0): await client0.query(r''' set_type('X', { a: new_type('A'), @@ -1686,7 +1681,7 @@ async def _TODO_test_init_create(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_rename(self, client0): + async def test_rename(self, client0): await client0.query(r''' set_type('A', {i: 'int'}); set_type('B', {a: 'A'}); @@ -1742,7 +1737,7 @@ async def _TODO_test_rename(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_rename_prop(self, client0): + async def test_rename_prop(self, client0): await client0.query(r''' set_type('Test', { arr: '[]', @@ -1795,7 +1790,7 @@ async def _TODO_test_rename_prop(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_wrap(self, client0): + async def test_wrap(self, client0): await client0.query(r''' new_type('_A'); // true through mod_type new_type('_B'); // true through set_type @@ -1836,7 +1831,7 @@ async def _TODO_test_wrap(self, client0): client1.close() await client1.wait_closed() - async def _TODO_test_same_deep(self, client): + async def test_same_deep(self, client): res = await client.query(r"""//ti new_type('User'); set_type('User', { @@ -1901,7 +1896,7 @@ async def _TODO_test_same_deep(self, client): "nosamedeep": {}, }) - async def _TODO_test_type_name_id(self, client): + async def test_type_name_id(self, client): with self.assertRaisesRegex( LookupError, r'multiple Id \(\'#\'\) definitions on type `User`'): @@ -2024,7 +2019,7 @@ async def _TODO_test_type_name_id(self, client): client1.close() await client1.wait_closed() - async def _TODO_test_prefix_flags(self, client): + async def test_prefix_flags(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `name` on type `User`; ' @@ -2112,7 +2107,7 @@ async def _TODO_test_prefix_flags(self, client): }, }) - async def _TODO_test_rename_type_with_flags(self, client): + async def test_rename_type_with_flags(self, client): res = await client.query(r"""//ti new_type('A'); set_type('B', { @@ -2136,7 +2131,7 @@ async def _TODO_test_rename_type_with_flags(self, client): [['a0', 'AA'], ['a1', 'AA?'], ['a2', '^AA'], ['a3', '!?&+^[AA?]?']] ]) - async def _TODO_test_email(self, client): + async def test_email(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `def` on type `E`; ' @@ -2185,7 +2180,7 @@ async def _TODO_test_email(self, client): E{}.email = 'abc'; """) - async def _TODO_test_url(self, client): + async def test_url(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `def` on type `E`; ' @@ -2234,7 +2229,7 @@ async def _TODO_test_url(self, client): E{}.url = 'abc'; """) - async def _TODO_test_tel(self, client): + async def test_tel(self, client): with self.assertRaisesRegex( ValueError, r'invalid declaration for `def` on type `E`; ' diff --git a/src/ti/commits.c b/src/ti/commits.c index f4ef2847..69afdeae 100644 --- a/src/ti/commits.c +++ b/src/ti/commits.c @@ -172,7 +172,7 @@ static int commits__options( } ex_set(w->e, EX_VALUE_ERROR, "invalid option `%.*s`; valid options: `scope`, `contains`, " - "`match`, `id`, `first`, `last`, `before`, `after`%s", + "`match`, `id`, `first`, `last`, `before`, `after`, `has_err`%s", key->n, key->data, w->allow_detail ? ", `detail`" : ""); return 0; @@ -199,10 +199,12 @@ int ti_commits_options( !options->first && !options->last && !options->before && - !options->after) + !options->after && + !options->has_err) ex_set(e, EX_LOOKUP_ERROR, "at least one of the following options must be set: " - "`contains`, `match`, `id`, `first`, `last`, `before`, `after`"); + "`contains`, `match`, `id`, `first`, `last`, `before`, `after`" + ", `has_err`"DOC_HISTORY""DOC_DEL_HISTORY); return e->nr; } @@ -232,9 +234,9 @@ int ti_commits_history( history->access = &ti.access_thingsdb; break; case TI_SCOPE_NODE: - history->commits = NULL; - history->access = NULL; - break; + ex_set(e, EX_OPERATION, + "commit history is not supported by `@node` scopes"); + return e->nr; case TI_SCOPE_COLLECTION: { ti_collection_t * collection = ti_collections_get_by_strn( @@ -256,7 +258,7 @@ int ti_commits_history( } if (!(*history->commits)) ex_set(e, EX_OPERATION, - "history is not enabled for the `*.*s` scope", + "commit history is not enabled for the `%.*s` scope", options->scope->n, options->scope->data); return e->nr; @@ -269,8 +271,9 @@ int ti_commits_history( if (*history->commits) { ex_set(e, EX_LOOKUP_ERROR, - "multiple scopes with `history` enabled; " - "use the `scope` option to specify an explicit scope"); + "more than one scope has commit history enabled; you must " + "use the `scope` option to tell ThingsDB which scope " + "to use"DOC_HISTORY""DOC_DEL_HISTORY); return e->nr; } history->commits = &collection->commits; @@ -281,7 +284,7 @@ int ti_commits_history( if (!(*history->commits)) ex_set(e, EX_LOOKUP_ERROR, - "no scope found with `history` enabled"DOC_SET_HISTORY); + "no scope found with commit history enabled"DOC_SET_HISTORY); return e->nr; } @@ -296,7 +299,8 @@ vec_t ** ti_commits_from_scope(ti_raw_t * scope, ex_t * e) case TI_SCOPE_THINGSDB: return &ti.commits; case TI_SCOPE_NODE: - ex_set(e, EX_LOOKUP_ERROR, "no commit history in node scope"); + ex_set(e, EX_OPERATION, + "commit history is not supported by `@node` scopes"); return NULL; case TI_SCOPE_COLLECTION: { @@ -309,6 +313,7 @@ vec_t ** ti_commits_from_scope(ti_raw_t * scope, ex_t * e) ex_set(e, EX_LOOKUP_ERROR, "collection `%.*s` not found", scope_.via.collection_name.sz, scope_.via.collection_name.name); + return NULL; } } ex_set_internal(e); @@ -360,6 +365,9 @@ static _Bool commits__match( if (options->id && commit->id != (uint64_t) VINT(options->id)) return false; + if (options->has_err && !commit->err_msg != options->has_err->bool_) + return false; + return true; } From 958588f48bbcdc934a6c875cb48d2ba797744a69 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 13:47:55 +0100 Subject: [PATCH 13/23] tests update --- inc/ti/fn/fncommit.h | 5 +- inc/ti/fn/fndelhistory.h | 14 ++-- itest/test_commits.py | 146 ++++++++++++++++++++++++++++++++++++--- src/ti.c | 1 + src/ti/commits.c | 23 ++++-- 5 files changed, 167 insertions(+), 22 deletions(-) diff --git a/inc/ti/fn/fncommit.h b/inc/ti/fn/fncommit.h index 30ee676f..8656e655 100644 --- a/inc/ti/fn/fncommit.h +++ b/inc/ti/fn/fncommit.h @@ -19,9 +19,8 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) if (!(*commits)) { ex_set(e, EX_OPERATION, - "commit history is not enabled for the `%s` scope" - DOC_SET_HISTORY, - ti_query_scope_name(query)); + "commit history is not enabled" + DOC_SET_HISTORY); goto fail0; } diff --git a/inc/ti/fn/fndelhistory.h b/inc/ti/fn/fndelhistory.h index fe8ba8ea..d0de2afb 100644 --- a/inc/ti/fn/fndelhistory.h +++ b/inc/ti/fn/fndelhistory.h @@ -23,12 +23,14 @@ static int do__f_del_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) if (!deleted) goto fail; - task = ti_task_get_task( - query->change, - query->collection ? query->collection->root : ti.thing0); - if (!task || ti_task_add_del_history(task, history.scope_id, deleted)) - goto undo; - + if (deleted->n) + { + task = ti_task_get_task( + query->change, + query->collection ? query->collection->root : ti.thing0); + if (!task || ti_task_add_del_history(task, history.scope_id, deleted)) + goto undo; + } ti_val_unsafe_drop(query->rval); query->rval = (ti_val_t *) ti_vint_create((int64_t) deleted->n); if (!query->rval) diff --git a/itest/test_commits.py b/itest/test_commits.py index 2cda52e4..9be95f1a 100755 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -220,7 +220,7 @@ async def test_history(self, q0, q1): with self.assertRaisesRegex( LookupError, - r'xxx'): + r'collection `s` not found'): await q0("""//ti history({scope: '//s', last:0}); """, scope='/t') @@ -230,6 +230,22 @@ async def test_history(self, q0, q1): set_history('/t', true); """, scope='/t') + with self.assertRaisesRegex( + ValueError, + r'option `last` expects an integer value greater ' + r'than or equal to 0'): + await q0("""//ti + history({scope: '/t', last:-1}); + """, scope='/t') + + with self.assertRaisesRegex( + TypeError, + r'expecting `last` to be of type `int` but got ' + r'type `bool` instead'): + await q0("""//ti + history({scope: '/t', last:false}); + """, scope='/t') + with self.assertRaisesRegex( LookupError, r'more than one scope has commit history enabled; you must ' @@ -248,6 +264,50 @@ async def test_history(self, q0, q1): history({scope: '//stuff'}); """, scope='/t') + with self.assertRaisesRegex( + ValueError, + r'invalid option `x` was provided; valid options are: ' + r'`scope`, `contains`, `match`, `id`, `first`, `last`, ' + r'`before`, `after`, `has_err`, `detail`'): + await q0("""//ti + history({scope: '//stuff', x: ''}); + """, scope='/t') + + async def test_del_history(self, q0, q1): + await q0("""//ti + set_history('//stuff', false); + set_history('/t', false); + """, scope='/t') + + with self.assertRaisesRegex( + LookupError, + r'function `del_history` is undefined in the `@node` scope; ' + r'you might want to query the `@thingsdb` scope\?'): + await q0("""//ti + del_history({last:0}); + """, scope='/n') + + with self.assertRaisesRegex( + LookupError, + r'no scope found with commit history enabled;'): + await q0("""//ti + del_history({last:0}); + """, scope='/t') + + await q0("""//ti + set_history('//stuff', true); + set_history('/t', true); + """, scope='/t') + + with self.assertRaisesRegex( + ValueError, + r'invalid option `x` was provided; valid options are: ' + r'`scope`, `contains`, `match`, `id`, `first`, `last`, ' + r'`before`, `after`, `has_err`$'): + await q0("""//ti + del_history({scope: '//stuff', x: ''}); + """, scope='/t') + async def test_commit(self, q0, q1): await q0("""//ti set_history('//stuff', false); @@ -255,38 +315,106 @@ async def test_commit(self, q0, q1): """, scope='/t') with self.assertRaisesRegex( LookupError, - r'xxxx'): + r'function `commit` is undefined in the `@node` scope; ' + r'you might want to query the `@thingsdb` or ' + r'a `@collection` scope\?'): await q0("""//ti commit('Test'); """, scope='/n') with self.assertRaisesRegex( - LookupError, - r'xxxx'): + OperationError, + r'commit history is not enabled'): await q0("""//ti commit('Test'); new_type('A'); """) + + await q0("""//ti + set_history('//stuff', true); + set_history('/t', true); + """, scope='/t') + with self.assertRaisesRegex( NumArgumentsError, - r'xxxx'): + r'function `commit` takes 1 argument but 0 were given'): await q0("""//ti commit(); new_type('A'); """) with self.assertRaisesRegex( - NumArgumentsError, - r'xxxx'): + TypeError, + r'function `commit` expects argument 1 to be of type `str` ' + r'but got type `int` instead'): await q0("""//ti commit(123); new_type('A'); """) with self.assertRaisesRegex( - NumArgumentsError, - r'xxxx'): + OperationError, + r'commit without a change;'): await q0("""//ti commit('Test'); """) + for i in range(10): + await q0("""//ti + commit(`Create type T{i}`); + set_type(`T{i}`, {name: 'str'}); + """, i=i) + + await asyncio.sleep(3.0) + + for i in range(10): + await q0("""//ti + commit(`Add x to type T{i}`); + mod_type(`T{i}`, 'add', 'x', 'int'); + """, i=i) + + before, after, both, deleted = await q0("""//ti + wse(); + moment = datetime().move('seconds', -2); + [ + history({ + scope: '//stuff', + before: moment, + }), + history({ + scope: '//stuff', + after: moment, + }), + history({ + scope: '//stuff', + before: moment, + after: moment, + }), + del_history({ + scope: '//stuff', + before: moment, + after: moment, + }), + ]; + """, scope='/t') + + self.assertEqual(len(before), 10) + self.assertEqual(len(after), 10) + self.assertEqual(len(both), 0) + self.assertEqual(deleted, 0) + + self.assertTrue(before[0]['id'] < after[0]['id']) + + with self.assertRaisesRegex( + OperationError, + r'function `to_type` requires a commit before it can be used ' + r'on the `root\(\)` of a `@collection` scope with commit ' + r'history enabled;'): + await q0("""//ti + .to_type('T0'); + """) + + + + + diff --git a/src/ti.c b/src/ti.c index 4a73d092..f3824f0d 100644 --- a/src/ti.c +++ b/src/ti.c @@ -165,6 +165,7 @@ void ti_destroy(void) vec_destroy(ti.access_node, (vec_destroy_cb) ti_auth_destroy); vec_destroy(ti.access_thingsdb, (vec_destroy_cb) ti_auth_destroy); + vec_destroy(ti.commits, (vec_destroy_cb) ti_commit_destroy); smap_destroy(ti.procedures, (smap_destroy_cb) ti_procedure_destroy); /* remove late since counters can be updated */ diff --git a/src/ti/commits.c b/src/ti/commits.c index 69afdeae..8f0b5727 100644 --- a/src/ti/commits.c +++ b/src/ti/commits.c @@ -90,6 +90,13 @@ static int commits__options( ti_val_str(val)); return w->e->nr; } + if (VINT(val) < 0) + { + ex_set(w->e, EX_VALUE_ERROR, + "option `last` expects an " + "integer value greater than or equal to 0"); + return w->e->nr; + } w->options->last = (ti_vint_t *) val; return 0; } @@ -105,6 +112,13 @@ static int commits__options( ti_val_str(val)); return w->e->nr; } + if (VINT(val) < 0) + { + ex_set(w->e, EX_VALUE_ERROR, + "option `first` expects an " + "integer value greater than or equal to 0"); + return w->e->nr; + } w->options->first = (ti_vint_t *) val; return 0; } @@ -171,11 +185,12 @@ static int commits__options( return 0; } ex_set(w->e, EX_VALUE_ERROR, - "invalid option `%.*s`; valid options: `scope`, `contains`, " - "`match`, `id`, `first`, `last`, `before`, `after`, `has_err`%s", + "invalid option `%.*s` was provided; valid options are: `scope`, " + "`contains`, `match`, `id`, `first`, `last`, `before`, " + "`after`, `has_err`%s", key->n, key->data, w->allow_detail ? ", `detail`" : ""); - return 0; + return w->e->nr; } int ti_commits_options( @@ -391,7 +406,7 @@ static void commits__init( } if (options->after) { - uint32_t i = *end - 1; + uint32_t i = commits->n; for (vec_each_rev(commits, ti_commit_t, commit) --i) if (commit->ts <= options->after->ts) break; From 24143f3be56544c26c40d456f37b393eaa6bbe83 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 14:38:25 +0100 Subject: [PATCH 14/23] no search in title --- itest/test_commits.py | 131 +++++++++++++++++++++++++++++++----------- src/ti/commits.c | 20 ++++++- 2 files changed, 113 insertions(+), 38 deletions(-) diff --git a/itest/test_commits.py b/itest/test_commits.py index 9be95f1a..788a33b1 100755 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -238,6 +238,13 @@ async def test_history(self, q0, q1): history({scope: '/t', last:-1}); """, scope='/t') + with self.assertRaisesRegex( + ValueError, + r'option `contains` must be a non empty string'): + await q0("""//ti + history({scope: '/t', contains:""}); + """, scope='/t') + with self.assertRaisesRegex( TypeError, r'expecting `last` to be of type `int` but got ' @@ -357,50 +364,73 @@ async def test_commit(self, q0, q1): """) for i in range(10): - await q0("""//ti - commit(`Create type T{i}`); - set_type(`T{i}`, {name: 'str'}); - """, i=i) + # use f-string format as we want T0..9 be part of the code + await q0(f"""//ti + commit('Create type T{i}'); + set_type('T{i}', {{name: 'str'}}); + """) await asyncio.sleep(3.0) for i in range(10): - await q0("""//ti - commit(`Add x to type T{i}`); - mod_type(`T{i}`, 'add', 'x', 'int'); - """, i=i) + # use f-string format as we want T0..9 be part of the code + await q0(f"""//ti + commit('Add x to type T{i}'); + mod_type('T{i}', 'add', 'x', 'int'); + """) - before, after, both, deleted = await q0("""//ti - wse(); - moment = datetime().move('seconds', -2); - [ - history({ - scope: '//stuff', - before: moment, - }), - history({ - scope: '//stuff', - after: moment, - }), - history({ - scope: '//stuff', - before: moment, - after: moment, - }), - del_history({ - scope: '//stuff', - before: moment, - after: moment, - }), - ]; - """, scope='/t') + before, b5, after, a5, both, deleted, t0m, t0c = await q0("""//ti + wse(); + moment = datetime().move('seconds', -2); + [ + history({ + scope: '//stuff', + before: moment, + }), + history({ + scope: '//stuff', + before: moment, + last: 5, + }), + history({ + scope: '//stuff', + after: moment, + }), + history({ + scope: '//stuff', + after: moment, + first: 5, + }), + history({ + scope: '//stuff', + before: moment, + after: moment, + }), + del_history({ + scope: '//stuff', + before: moment, + after: moment, + }), + history({ + scope: '//stuff', + match: /\\bT0\\b/m, + }), + history({ + scope: '//stuff', + contains: 'T0' + }), + ]; + """, scope='/t') self.assertEqual(len(before), 10) self.assertEqual(len(after), 10) self.assertEqual(len(both), 0) + self.assertEqual(len(t0m), 2) self.assertEqual(deleted, 0) - + self.assertEqual(t0m, t0c) self.assertTrue(before[0]['id'] < after[0]['id']) + self.assertEqual(before[-5:], b5) + self.assertEqual(after[:5], a5) with self.assertRaisesRegex( OperationError, @@ -411,11 +441,42 @@ async def test_commit(self, q0, q1): .to_type('T0'); """) + try: + await q0("""//ti + commit(`Add another type T0 (should fail)`); + new_type('T0'); + """) + except Exception as e: + err_msg = str(e) + else: + err_msg = None + self.assertTrue(isinstance(err_msg, str)) + await self.node0.shutdown() + await self.node0.run() - - + for q in (q0, q1): + with_err, no_err, both = await q("""//ti + [ + history({ + scope: '//stuff', + has_err: true, + }), + history({ + scope: '//stuff', + has_err: false, + }), + history({ + scope: '//stuff', + match: /.*/, + }), + ]; + """, scope='/t') + self.assertEqual(len(with_err), 1) + self.assertEqual(len(no_err), 20) + self.assertEqual(len(both), 21) + self.assertEqual(with_err[0]['err_msg'], err_msg) if __name__ == '__main__': diff --git a/src/ti/commits.c b/src/ti/commits.c index 8f0b5727..27670546 100644 --- a/src/ti/commits.c +++ b/src/ti/commits.c @@ -45,6 +45,12 @@ static int commits__options( ti_val_str(val)); return w->e->nr; } + if (((ti_raw_t *) val)->n == 0) + { + ex_set(w->e, EX_VALUE_ERROR, + "option `contains` must be a non empty string"); + return w->e->nr; + } w->options->contains = (ti_raw_t *) val; return 0; } @@ -75,6 +81,13 @@ static int commits__options( ti_val_str(val)); return w->e->nr; } + if (VINT(val) < 0) + { + ex_set(w->e, EX_VALUE_ERROR, + "option `id` expects an " + "integer value greater than or equal to 0"); + return w->e->nr; + } w->options->id = (ti_vint_t *) val; return 0; } @@ -358,9 +371,11 @@ static _Bool commits__match( ti_commit_t * commit, ti_commits_options_t * options) { + /* For both contains and match, we do not scan the message as the + * message is most likely part of the code. This may not be perfect when + * the message is using a construction but we accept this trade-off */ if (options->contains && ( !ti_raw_contains(commit->code, options->contains) && - !ti_raw_contains(commit->message, options->contains) && !ti_raw_contains(commit->by, options->contains) && ( !commit->err_msg || !ti_raw_contains(commit->err_msg, options->contains) @@ -369,7 +384,6 @@ static _Bool commits__match( return false; if (options->match && ( !ti_regex_test(options->match, commit->code) && - !ti_regex_test(options->match, commit->message) && !ti_regex_test(options->match, commit->by) && ( !commit->err_msg || !ti_regex_test(options->match, commit->err_msg) @@ -380,7 +394,7 @@ static _Bool commits__match( if (options->id && commit->id != (uint64_t) VINT(options->id)) return false; - if (options->has_err && !commit->err_msg != options->has_err->bool_) + if (options->has_err && !!commit->err_msg != options->has_err->bool_) return false; return true; From 9bc6203885a19053242f1a57e14e092e842db3e1 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 14:41:20 +0100 Subject: [PATCH 15/23] upd test --- itest/test_commits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itest/test_commits.py b/itest/test_commits.py index 788a33b1..c8c3fe9d 100755 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -413,7 +413,7 @@ async def test_commit(self, q0, q1): }), history({ scope: '//stuff', - match: /\\bT0\\b/m, + match: /\\bT0\\b/, }), history({ scope: '//stuff', From f4fc113748a9b87d0f4240d9c3a09de82247e10f Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 15:10:21 +0100 Subject: [PATCH 16/23] del history fix --- inc/ti/fn/fnhistory.h | 18 +++++++++++--- itest/test_commits.py | 57 +++++++++++++++++++++++++++++++++++++++++++ src/ti/commits.c | 4 +-- 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/inc/ti/fn/fnhistory.h b/inc/ti/fn/fnhistory.h index 25e484b3..f0b3112f 100644 --- a/inc/ti/fn/fnhistory.h +++ b/inc/ti/fn/fnhistory.h @@ -35,9 +35,21 @@ static int do__f_history(ti_query_t * query, cleri_node_t * nd, ex_t * e) } ti_val_unsafe_drop(query->rval); - query->rval = (ti_val_t *) ti_varr_from_vec(filtered); - if (!query->rval) - goto fail; + if (options.id) + { + assert(filtered->n <= 1); + + query->rval = (ti_val_t *) (filtered->n + ? VEC_first(filtered) + : ti_nil_get()); + vec_destroy(filtered, NULL); + } + else + { + query->rval = (ti_val_t *) ti_varr_from_vec(filtered); + if (!query->rval) + goto fail; + } return e->nr; fail: diff --git a/itest/test_commits.py b/itest/test_commits.py index c8c3fe9d..a206c8d9 100755 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -478,6 +478,63 @@ async def test_commit(self, q0, q1): self.assertEqual(len(both), 21) self.assertEqual(with_err[0]['err_msg'], err_msg) + commit_id = with_err[0]['id'] + + commit = await q0("""//ti + history({ + scope: '//stuff', + id: commit_id, + detail: true, + }); + """, commit_id=commit_id, scope='/t') + + self.assertEqual(commit['id'], commit_id) + self.assertTrue('code' in commit) + self.assertTrue('should fail' in commit['code']) + + n = await q0("""//ti + del_history({ + scope: '//stuff', + first: 3, + }); + """, commit_id=commit_id, scope='/t') + self.assertEqual(n, 3) + n = await q0("""//ti + del_history({ + scope: '//stuff', + last: 3, + }); + """, commit_id=commit_id, scope='/t') + self.assertEqual(n, 3) + + for q in (q0, q1): + t0, t4, t9, n = await q("""//ti + wse(); + [ + history({ + scope: '//stuff', + contains: 'T0', + }), + history({ + scope: '//stuff', + contains: 'T4', + }), + history({ + scope: '//stuff', + contains: 'T9', + }), + collection_info('stuff').load().commit_history, + ]; + """, scope='/t') + self.assertEqual(len(t0), 1) + self.assertEqual(len(t4), 2) + self.assertEqual(len(t9), 1) + self.assertEqual(n, 15) + + + + + if __name__ == '__main__': run_test(TestCommits()) diff --git a/src/ti/commits.c b/src/ti/commits.c index 27670546..334e142b 100644 --- a/src/ti/commits.c +++ b/src/ti/commits.c @@ -465,9 +465,9 @@ vec_t * ti_commits_del(vec_t ** commits, ti_commits_options_t * options) return NULL; commits__init(*commits, options, &begin, &end); - while (begin <= --end) + while (begin < end) { - ti_commit_t * commit = VEC_get(*commits, end); + ti_commit_t * commit = VEC_get(*commits, --end); if (commits__match(commit, options) && vec_push(&vec, vec_remove(*commits, end))) goto fail; From e349c3d0ff7be0f580869b0d55d83261abe42d9c Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 15:18:34 +0100 Subject: [PATCH 17/23] revert limit --- CMakeLists.txt | 2 +- inc/ti/fn/fnmodtype.h | 2 +- inc/ti/names.h | 4 +- src/ti/names.c | 164 +++++++++++++++++++++--------------------- src/ti/val.c | 40 +++++------ 5 files changed, 105 insertions(+), 107 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e21e6e6..2923ff8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,7 +48,7 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") ) else() # Set these flags for other than MacOS build - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -finline-limit=6000") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -finline-limit=4000") if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2") if($ENV{LEGACY}) diff --git a/inc/ti/fn/fnmodtype.h b/inc/ti/fn/fnmodtype.h index 90659345..a6ddb551 100644 --- a/inc/ti/fn/fnmodtype.h +++ b/inc/ti/fn/fnmodtype.h @@ -1547,7 +1547,7 @@ static int do__f_mod_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) fn_arg_name_check("mod_type", DOC_MOD_TYPE, 3, query->rval, e)) goto fail1; - name = ti_names_from_raw_slow((ti_raw_t *) query->rval); + name = ti_names_from_raw((ti_raw_t *) query->rval); if (!name) { ex_set_mem(e); diff --git a/inc/ti/names.h b/inc/ti/names.h index 9bc8b76e..269f6dd7 100644 --- a/inc/ti/names.h +++ b/inc/ti/names.h @@ -40,11 +40,9 @@ static inline ti_name_t * ti_names_get(const char * str, size_t n) ti_name_t * ti_names_get_slow(const char * str, size_t n); -#define ti_names_from_str(str__) ti_names_get((str__), strlen(str__)) #define ti_names_from_raw(raw__) \ ti_names_get((const char *) (raw__)->data, (raw__)->n) -#define ti_names_from_raw_slow(raw__) \ - ti_names_get_slow((const char *) (raw__)->data, (raw__)->n) +#define ti_names_from_str_slow(str__) ti_names_get_slow((str__), strlen(str__)) static inline ti_name_t * ti_names_weak_from_raw(ti_raw_t * raw) { diff --git a/src/ti/names.c b/src/ti/names.c index 53f21944..48bf9cae 100644 --- a/src/ti/names.c +++ b/src/ti/names.c @@ -45,89 +45,89 @@ void ti_names_destroy(void) */ void ti_names_inject_common(void) { - (void) ti_names_from_str("_"); - (void) ti_names_from_str("a"); - (void) ti_names_from_str("b"); - (void) ti_names_from_str("c"); - (void) ti_names_from_str("x"); - (void) ti_names_from_str("y"); - (void) ti_names_from_str("z"); - (void) ti_names_from_str("tmp"); + (void) ti_names_from_str_slow("_"); + (void) ti_names_from_str_slow("a"); + (void) ti_names_from_str_slow("b"); + (void) ti_names_from_str_slow("c"); + (void) ti_names_from_str_slow("x"); + (void) ti_names_from_str_slow("y"); + (void) ti_names_from_str_slow("z"); + (void) ti_names_from_str_slow("tmp"); #ifdef TI_PRELOAD_NAMES - (void) ti_names_from_str("access"); - (void) ti_names_from_str("archive_files"); - (void) ti_names_from_str("archived_in_memory"); - (void) ti_names_from_str("arguments"); - (void) ti_names_from_str("cache_expiration_time"); - (void) ti_names_from_str("cached_names"); - (void) ti_names_from_str("cached_queries"); - (void) ti_names_from_str("client_port"); - (void) ti_names_from_str("collection_id"); - (void) ti_names_from_str("conf"); - (void) ti_names_from_str("connected_clients"); - (void) ti_names_from_str("created_at"); - (void) ti_names_from_str("created_on"); - (void) ti_names_from_str("db_stored_change_id"); - (void) ti_names_from_str("default"); - (void) ti_names_from_str("definition"); - (void) ti_names_from_str("doc"); - (void) ti_names_from_str("enum_id"); - (void) ti_names_from_str("events_in_queue"); - (void) ti_names_from_str("expiration_time"); - (void) ti_names_from_str("fields"); - (void) ti_names_from_str("file_template"); - (void) ti_names_from_str("file"); - (void) ti_names_from_str("files"); - (void) ti_names_from_str("global_committed_change_id"); - (void) ti_names_from_str("global_stored_change_id"); - (void) ti_names_from_str("has_password"); - (void) ti_names_from_str("http_api_port"); - (void) ti_names_from_str("http_status_port"); - (void) ti_names_from_str("id"); - (void) ti_names_from_str("ip_support"); - (void) ti_names_from_str("key"); - (void) ti_names_from_str("libcleri_version"); - (void) ti_names_from_str("libpcre2_version"); - (void) ti_names_from_str("libuv_version"); - (void) ti_names_from_str("local_committed_change_id"); - (void) ti_names_from_str("local_stored_change_id"); - (void) ti_names_from_str("log_level"); - (void) ti_names_from_str("max_files"); - (void) ti_names_from_str("members"); - (void) ti_names_from_str("methods"); - (void) ti_names_from_str("modified_at"); - (void) ti_names_from_str("msgpack_version"); - (void) ti_names_from_str("name"); - (void) ti_names_from_str("next_free_id"); - (void) ti_names_from_str("next_run"); - (void) ti_names_from_str("next_thing_id"); - (void) ti_names_from_str("node_id"); - (void) ti_names_from_str("node_name"); - (void) ti_names_from_str("node_port"); - (void) ti_names_from_str("privileges"); - (void) ti_names_from_str("relations"); - (void) ti_names_from_str("repeat"); - (void) ti_names_from_str("restarts"); - (void) ti_names_from_str("result_code"); - (void) ti_names_from_str("result_message"); - (void) ti_names_from_str("result_size_limit"); - (void) ti_names_from_str("scheduled_backups"); - (void) ti_names_from_str("scope"); - (void) ti_names_from_str("status"); - (void) ti_names_from_str("storage_path"); - (void) ti_names_from_str("syntax_version"); - (void) ti_names_from_str("tasks"); - (void) ti_names_from_str("threshold_query_cache"); - (void) ti_names_from_str("tokens"); - (void) ti_names_from_str("type_id"); - (void) ti_names_from_str("uptime"); - (void) ti_names_from_str("user_id"); - (void) ti_names_from_str("user"); - (void) ti_names_from_str("version"); - (void) ti_names_from_str("with_side_effects"); - (void) ti_names_from_str("wrap_only"); - (void) ti_names_from_str("yajl_version"); - (void) ti_names_from_str("zone"); + (void) ti_names_from_str_slow("access"); + (void) ti_names_from_str_slow("archive_files"); + (void) ti_names_from_str_slow("archived_in_memory"); + (void) ti_names_from_str_slow("arguments"); + (void) ti_names_from_str_slow("cache_expiration_time"); + (void) ti_names_from_str_slow("cached_names"); + (void) ti_names_from_str_slow("cached_queries"); + (void) ti_names_from_str_slow("client_port"); + (void) ti_names_from_str_slow("collection_id"); + (void) ti_names_from_str_slow("conf"); + (void) ti_names_from_str_slow("connected_clients"); + (void) ti_names_from_str_slow("created_at"); + (void) ti_names_from_str_slow("created_on"); + (void) ti_names_from_str_slow("db_stored_change_id"); + (void) ti_names_from_str_slow("default"); + (void) ti_names_from_str_slow("definition"); + (void) ti_names_from_str_slow("doc"); + (void) ti_names_from_str_slow("enum_id"); + (void) ti_names_from_str_slow("events_in_queue"); + (void) ti_names_from_str_slow("expiration_time"); + (void) ti_names_from_str_slow("fields"); + (void) ti_names_from_str_slow("file_template"); + (void) ti_names_from_str_slow("file"); + (void) ti_names_from_str_slow("files"); + (void) ti_names_from_str_slow("global_committed_change_id"); + (void) ti_names_from_str_slow("global_stored_change_id"); + (void) ti_names_from_str_slow("has_password"); + (void) ti_names_from_str_slow("http_api_port"); + (void) ti_names_from_str_slow("http_status_port"); + (void) ti_names_from_str_slow("id"); + (void) ti_names_from_str_slow("ip_support"); + (void) ti_names_from_str_slow("key"); + (void) ti_names_from_str_slow("libcleri_version"); + (void) ti_names_from_str_slow("libpcre2_version"); + (void) ti_names_from_str_slow("libuv_version"); + (void) ti_names_from_str_slow("local_committed_change_id"); + (void) ti_names_from_str_slow("local_stored_change_id"); + (void) ti_names_from_str_slow("log_level"); + (void) ti_names_from_str_slow("max_files"); + (void) ti_names_from_str_slow("members"); + (void) ti_names_from_str_slow("methods"); + (void) ti_names_from_str_slow("modified_at"); + (void) ti_names_from_str_slow("msgpack_version"); + (void) ti_names_from_str_slow("name"); + (void) ti_names_from_str_slow("next_free_id"); + (void) ti_names_from_str_slow("next_run"); + (void) ti_names_from_str_slow("next_thing_id"); + (void) ti_names_from_str_slow("node_id"); + (void) ti_names_from_str_slow("node_name"); + (void) ti_names_from_str_slow("node_port"); + (void) ti_names_from_str_slow("privileges"); + (void) ti_names_from_str_slow("relations"); + (void) ti_names_from_str_slow("repeat"); + (void) ti_names_from_str_slow("restarts"); + (void) ti_names_from_str_slow("result_code"); + (void) ti_names_from_str_slow("result_message"); + (void) ti_names_from_str_slow("result_size_limit"); + (void) ti_names_from_str_slow("scheduled_backups"); + (void) ti_names_from_str_slow("scope"); + (void) ti_names_from_str_slow("status"); + (void) ti_names_from_str_slow("storage_path"); + (void) ti_names_from_str_slow("syntax_version"); + (void) ti_names_from_str_slow("tasks"); + (void) ti_names_from_str_slow("threshold_query_cache"); + (void) ti_names_from_str_slow("tokens"); + (void) ti_names_from_str_slow("type_id"); + (void) ti_names_from_str_slow("uptime"); + (void) ti_names_from_str_slow("user_id"); + (void) ti_names_from_str_slow("user"); + (void) ti_names_from_str_slow("version"); + (void) ti_names_from_str_slow("with_side_effects"); + (void) ti_names_from_str_slow("wrap_only"); + (void) ti_names_from_str_slow("yajl_version"); + (void) ti_names_from_str_slow("zone"); #endif } diff --git a/src/ti/val.c b/src/ti/val.c index 3fdfb099..652ef95d 100644 --- a/src/ti/val.c +++ b/src/ti/val.c @@ -756,26 +756,26 @@ int ti_val_init_common(void) "-_"); /* names */ - val__async_name = (ti_val_t *) ti_names_from_str("async"); - val__data_name = (ti_val_t *) ti_names_from_str("data"); - val__time_name = (ti_val_t *) ti_names_from_str("time"); - val__year_name = (ti_val_t *) ti_names_from_str("year"); - val__month_name = (ti_val_t *) ti_names_from_str("month"); - val__day_name = (ti_val_t *) ti_names_from_str("day"); - val__hour_name = (ti_val_t *) ti_names_from_str("hour"); - val__minute_name = (ti_val_t *) ti_names_from_str("minute"); - val__second_name = (ti_val_t *) ti_names_from_str("second"); - val__gmt_offset_name = (ti_val_t *) ti_names_from_str("gmt_offset"); - val__module_name = (ti_val_t *) ti_names_from_str("module"); - val__deep_name = (ti_val_t *) ti_names_from_str("deep"); - val__flags_name = (ti_val_t *) ti_names_from_str("flags"); - val__load_name = (ti_val_t *) ti_names_from_str("load"); - val__beautify_name = (ti_val_t *) ti_names_from_str("beautify"); - val__parent_name = (ti_val_t *) ti_names_from_str("parent"); - val__parent_type_name = (ti_val_t *) ti_names_from_str("parent_type"); - val__key_name = (ti_val_t *) ti_names_from_str("key"); - val__key_type_name = (ti_val_t *) ti_names_from_str("key_type"); - val__unnamed_name = (ti_val_t *) ti_names_from_str("__unnamed__"); + val__async_name = (ti_val_t *) ti_names_from_str_slow("async"); + val__data_name = (ti_val_t *) ti_names_from_str_slow("data"); + val__time_name = (ti_val_t *) ti_names_from_str_slow("time"); + val__year_name = (ti_val_t *) ti_names_from_str_slow("year"); + val__month_name = (ti_val_t *) ti_names_from_str_slow("month"); + val__day_name = (ti_val_t *) ti_names_from_str_slow("day"); + val__hour_name = (ti_val_t *) ti_names_from_str_slow("hour"); + val__minute_name = (ti_val_t *) ti_names_from_str_slow("minute"); + val__second_name = (ti_val_t *) ti_names_from_str_slow("second"); + val__gmt_offset_name = (ti_val_t *) ti_names_from_str_slow("gmt_offset"); + val__module_name = (ti_val_t *) ti_names_from_str_slow("module"); + val__deep_name = (ti_val_t *) ti_names_from_str_slow("deep"); + val__flags_name = (ti_val_t *) ti_names_from_str_slow("flags"); + val__load_name = (ti_val_t *) ti_names_from_str_slow("load"); + val__beautify_name = (ti_val_t *) ti_names_from_str_slow("beautify"); + val__parent_name = (ti_val_t *) ti_names_from_str_slow("parent"); + val__parent_type_name = (ti_val_t *) ti_names_from_str_slow("parent_type"); + val__key_name = (ti_val_t *) ti_names_from_str_slow("key"); + val__key_type_name = (ti_val_t *) ti_names_from_str_slow("key_type"); + val__unnamed_name = (ti_val_t *) ti_names_from_str_slow("__unnamed__"); val__re_email = (ti_val_t *) ti_regex_from_str("/^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\\.[a-zA-Z0-9-]+)*$/"); val__re_url = (ti_val_t *) ti_regex_from_str("/^(https?|ftp):\\/\\/[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)$/"); val__re_tel = (ti_val_t *) ti_regex_from_str("/^[\\+]?(\\([0-9]{1,4}\\)[-\\s\\.]?){0,2}([0-9]{1,4}[-\\s]?){3,5}$/"); From 58f04b60b7898f09bc743e51c30917ccd2d0ac04 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 16:29:09 +0100 Subject: [PATCH 18/23] fix inline, add recusion test export --- inc/ti/fn/fnmodtype.h | 2 +- inc/ti/names.h | 2 ++ itest/test_advanced.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/inc/ti/fn/fnmodtype.h b/inc/ti/fn/fnmodtype.h index a6ddb551..90659345 100644 --- a/inc/ti/fn/fnmodtype.h +++ b/inc/ti/fn/fnmodtype.h @@ -1547,7 +1547,7 @@ static int do__f_mod_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) fn_arg_name_check("mod_type", DOC_MOD_TYPE, 3, query->rval, e)) goto fail1; - name = ti_names_from_raw((ti_raw_t *) query->rval); + name = ti_names_from_raw_slow((ti_raw_t *) query->rval); if (!name) { ex_set_mem(e); diff --git a/inc/ti/names.h b/inc/ti/names.h index 269f6dd7..665f4fc1 100644 --- a/inc/ti/names.h +++ b/inc/ti/names.h @@ -42,6 +42,8 @@ ti_name_t * ti_names_get_slow(const char * str, size_t n); #define ti_names_from_raw(raw__) \ ti_names_get((const char *) (raw__)->data, (raw__)->n) +#define ti_names_from_raw_slow(raw__) \ + ti_names_get_slow((const char *) (raw__)->data, (raw__)->n) #define ti_names_from_str_slow(str__) ti_names_get_slow((str__), strlen(str__)) static inline ti_name_t * ti_names_weak_from_raw(ti_raw_t * raw) diff --git a/itest/test_advanced.py b/itest/test_advanced.py index feb1c513..8ff89bcb 100755 --- a/itest/test_advanced.py +++ b/itest/test_advanced.py @@ -2843,6 +2843,49 @@ async def test_return_no_args(self, client): \t}; }""") + async def test_recursive_enum_export(self, client): + q = client.query + await q("""//ti + t = {}; + set_enum('E', {A: t}); + E{A}.value().t = E{A}.value(); + """) + + script = await q("""//ti + export(); + """) + self.assertEqual(script, r''' +try(commit('Source: collection `stuff`')); + + +set_enum('E', { + A: {}, +}); + + + +mod_enum('E', 'mod', 'A', { + t: { + t: { + t: { + t: {} /* WARN: max deep reached */, + }, + }, + }, +}); + + + + +'DONE'; +'''.lstrip().replace(' ', '\t')) + await q("""//ti + new_collection('other') + """, scope='/t') + res = await q(script, scope='//other') + self.assertEqual(res, 'DONE') + + if __name__ == '__main__': run_test(TestAdvanced()) From cf948acf5025f3aabc3ad76f96de25398b84b5dc Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 16:33:23 +0100 Subject: [PATCH 19/23] Changelog and version --- CHANGELOG.md | 6 +++--- inc/ti/version.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3e78e2..b71250d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# v1.8.0-alpha8 +# v1.8.0-alpha9 * Added `TI_PROTO_CLIENT_REQ_EMIT_PEER` protocol, pr #414. * Changed `copy()` behavior for wrapped type, pr #415. @@ -9,12 +9,12 @@ * Include export enumerator members of type `thing`, pr #420. * Fixed missing "ID" (`#`) field in export, issue #421. * Added `type_all()` function, pr #422. -* Added commit history with new functions, pr#423. +* Fixed handling non-stored thing with `to_type()`, issue #424. +* Added commit history with new functions, pr #425 and discussion #423. - `commit()`: https://docs.thingsdb.io/v1/collection-api/commit/ - `history()`: https://docs.thingsdb.io/v1/thingsdb-api/history/ - `set_history()`: https://docs.thingsdb.io/v1/thingsdb-api/set_history/ - `del_history()`: https://docs.thingsdb.io/v1/thingsdb-api/del_history/ -* Fixed handling non-stored thing with `to_type()`, issue #424. # v1.7.6 diff --git a/inc/ti/version.h b/inc/ti/version.h index cbeee5f5..c8ff1680 100644 --- a/inc/ti/version.h +++ b/inc/ti/version.h @@ -25,7 +25,7 @@ * "-rc0" * "" */ -#define TI_VERSION_PRE_RELEASE "-alpha8" +#define TI_VERSION_PRE_RELEASE "-alpha9" #define TI_MAINTAINER \ "Jeroen van der Heijden " From b046eebca860d3d1a2eb231200a17e03803597fb Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 16:40:16 +0100 Subject: [PATCH 20/23] pep and ti commit scope test --- itest/test_advanced.py | 1 - itest/test_commits.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/itest/test_advanced.py b/itest/test_advanced.py index 8ff89bcb..2d791fcb 100755 --- a/itest/test_advanced.py +++ b/itest/test_advanced.py @@ -2886,6 +2886,5 @@ async def test_recursive_enum_export(self, client): self.assertEqual(res, 'DONE') - if __name__ == '__main__': run_test(TestAdvanced()) diff --git a/itest/test_commits.py b/itest/test_commits.py index a206c8d9..762c1a20 100755 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -531,9 +531,15 @@ async def test_commit(self, q0, q1): self.assertEqual(len(t9), 1) self.assertEqual(n, 15) + await q0("""//ti + commit('Test TI scope'); + new_user('A'); + """, scope='/t') - - + for q in (q0, q1): + await q('wse()') + n = await q('node_info().load().commit_history;', scope='/n') + self.assertEqual(n, 1) if __name__ == '__main__': From 99788c70b06e73c370ee2bcbecbbd85a9656ce9e Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 20:38:40 +0100 Subject: [PATCH 21/23] more testing --- inc/ti/fn/fncommit.h | 4 +++- itest/test_commits.py | 56 ++++++++++++++++++++++++++++++------------- src/ti/commits.c | 5 ++-- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/inc/ti/fn/fncommit.h b/inc/ti/fn/fncommit.h index 8656e655..35362dc1 100644 --- a/inc/ti/fn/fncommit.h +++ b/inc/ti/fn/fncommit.h @@ -26,7 +26,9 @@ static int do__f_commit(ti_query_t * query, cleri_node_t * nd, ex_t * e) switch ((ti_query_with_enum) query->with_tp) { - case TI_QUERY_WITH_PARSERES: break; + case TI_QUERY_WITH_PARSERES: + /* we use parseres, so it must be a query */ + break; case TI_QUERY_WITH_PROCEDURE: ex_set(e, EX_OPERATION, "function `commit` is not allowed in a procedure"); diff --git a/itest/test_commits.py b/itest/test_commits.py index 762c1a20..f0228538 100755 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -22,7 +22,7 @@ def with_node1(self): WARNING: Test requires a second node!!! ''') - @default_test_setup(num_nodes=1, seed=1, threshold_full_storage=10) + @default_test_setup(num_nodes=2, seed=1, threshold_full_storage=10) async def async_run(self): await self.node0.init_and_run() @@ -30,20 +30,18 @@ async def async_run(self): client0 = await get_client(self.node0) client0.set_default_scope('//stuff') - # add another node - if hasattr(self, 'node1'): - await self.node1.join_until_ready(client0) + await self.node1.join_until_ready(client0) - client1 = await get_client(self.node0) + client1 = await get_client(self.node1) client1.set_default_scope('//stuff') - await self.run_tests(client0.query, client1.query) + await self.run_tests(client0.query, client1.query, client0) for client in (client0, client1): client.close() await client.wait_closed() - async def test_set_history(self, q0, q1): + async def test_set_history(self, q0, q1, c0): with self.assertRaisesRegex( LookupError, r'function `set_history` is undefined in the `@node` scope; ' @@ -136,7 +134,7 @@ async def test_set_history(self, q0, q1): """, scope='/n') self.assertEqual(r, 'disabled') - async def test_access(self, q0, q1): + async def test_access(self, q0, q1, c0): await q0("""//ti new_collection('junk'); new_user('test1'); @@ -176,7 +174,7 @@ async def test_access(self, q0, q1): set_history('/t', false); """, scope='/t') - async def test_history(self, q0, q1): + async def test_history(self, q0, q1, c0): await q0("""//ti set_history('//stuff', false); set_history('/t', false); @@ -280,7 +278,7 @@ async def test_history(self, q0, q1): history({scope: '//stuff', x: ''}); """, scope='/t') - async def test_del_history(self, q0, q1): + async def test_del_history(self, q0, q1, c0): await q0("""//ti set_history('//stuff', false); set_history('/t', false); @@ -315,7 +313,7 @@ async def test_del_history(self, q0, q1): del_history({scope: '//stuff', x: ''}); """, scope='/t') - async def test_commit(self, q0, q1): + async def test_commit(self, q0, q1, c0): await q0("""//ti set_history('//stuff', false); set_history('/t', false); @@ -362,6 +360,15 @@ async def test_commit(self, q0, q1): await q0("""//ti commit('Test'); """) + with self.assertRaisesRegex( + OperationError, + r'function `commit` is not allowed in a future'): + await q0("""//ti + future(||{ + commit('Test'); + new_type('F'); + }); + """) for i in range(10): # use f-string format as we want T0..9 be part of the code @@ -373,11 +380,11 @@ async def test_commit(self, q0, q1): await asyncio.sleep(3.0) for i in range(10): - # use f-string format as we want T0..9 be part of the code - await q0(f"""//ti - commit('Add x to type T{i}'); - mod_type('T{i}', 'add', 'x', 'int'); - """) + # without f-string to ensure message + await q0("""//ti + commit(`Add x to type T{i}`); + mod_type(`T{i}`, 'add', 'x', 'int'); + """, i=i) before, b5, after, a5, both, deleted, t0m, t0c = await q0("""//ti wse(); @@ -541,6 +548,23 @@ async def test_commit(self, q0, q1): n = await q('node_info().load().commit_history;', scope='/n') self.assertEqual(n, 1) + async def test_from_procedure(self, q0, q1, c0): + await q0("""//ti + set_history('//stuff', true); + set_history('/t', false); + """, scope='/t') + await q0("""//ti + commit('add procedure...'); + new_procedure('test_commit', |msg| { + commit(msg); + wse(); + }); + """) + with self.assertRaisesRegex( + OperationError, + r'function `commit` is not allowed in a procedure'): + await c0.run('test_commit', 'test') + if __name__ == '__main__': run_test(TestCommits()) diff --git a/src/ti/commits.c b/src/ti/commits.c index 334e142b..4c9a45ae 100644 --- a/src/ti/commits.c +++ b/src/ti/commits.c @@ -371,10 +371,8 @@ static _Bool commits__match( ti_commit_t * commit, ti_commits_options_t * options) { - /* For both contains and match, we do not scan the message as the - * message is most likely part of the code. This may not be perfect when - * the message is using a construction but we accept this trade-off */ if (options->contains && ( + !ti_raw_contains(commit->message, options->contains) && !ti_raw_contains(commit->code, options->contains) && !ti_raw_contains(commit->by, options->contains) && ( !commit->err_msg || @@ -383,6 +381,7 @@ static _Bool commits__match( )) return false; if (options->match && ( + !ti_regex_test(options->match, commit->message) && !ti_regex_test(options->match, commit->code) && !ti_regex_test(options->match, commit->by) && ( !commit->err_msg || From 1a1f688fbc8ce33bd9ac83ed8036d781d8489542 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 20:45:54 +0100 Subject: [PATCH 22/23] rename and overreach test --- inc/ti/task.t.h | 2 +- itest/test_commits.py | 15 ++++++++++++--- src/ti/ctask.c | 4 ++-- src/ti/task.c | 2 +- src/ti/ttask.c | 4 ++-- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/inc/ti/task.t.h b/inc/ti/task.t.h index aa1103db..0e13f0f8 100644 --- a/inc/ti/task.t.h +++ b/inc/ti/task.t.h @@ -94,7 +94,7 @@ typedef enum TI_TASK_WHITELIST_DEL, /* 81 */ TI_TASK_SET_HISTORY, /* 82 */ TI_TASK_DEL_HISTORY, /* 83 */ - TI_TASK_COMMIT_ADD, /* 84 */ + TI_TASK_COMMIT, /* 84 */ } ti_task_enum; typedef struct ti_task_s ti_task_t; diff --git a/itest/test_commits.py b/itest/test_commits.py index f0228538..4a891cfb 100755 --- a/itest/test_commits.py +++ b/itest/test_commits.py @@ -464,7 +464,7 @@ async def test_commit(self, q0, q1, c0): await self.node0.run() for q in (q0, q1): - with_err, no_err, both = await q("""//ti + with_err, no_err, both0, both1, both2 = await q("""//ti [ history({ scope: '//stuff', @@ -478,11 +478,20 @@ async def test_commit(self, q0, q1, c0): scope: '//stuff', match: /.*/, }), - ]; + history({ + scope: '//stuff', + last: 9999, + }), + history({ + scope: '//stuff', + first: 9999, + }), ]; """, scope='/t') self.assertEqual(len(with_err), 1) self.assertEqual(len(no_err), 20) - self.assertEqual(len(both), 21) + self.assertEqual(len(both0), 21) + self.assertEqual(len(both1), 21) + self.assertEqual(len(both2), 21) self.assertEqual(with_err[0]['err_msg'], err_msg) commit_id = with_err[0]['id'] diff --git a/src/ti/ctask.c b/src/ti/ctask.c index cc03ab0f..573d9a99 100644 --- a/src/ti/ctask.c +++ b/src/ti/ctask.c @@ -3284,7 +3284,7 @@ int ctask__arr_remove(ti_thing_t * thing, mp_unp_t * up) * Returns 0 on success * - for example: '{name: closure}' */ -static int ctask__commit_add(ti_thing_t * thing, mp_unp_t * up) +static int ctask__commit(ti_thing_t * thing, mp_unp_t * up) { vec_t ** commits = &thing->collection->commits; ti_commit_t * commit = ti_commit_from_up(up); @@ -3422,7 +3422,7 @@ int ti_ctask_run(ti_thing_t * thing, mp_unp_t * up) case TI_TASK_WHITELIST_DEL: break; case TI_TASK_SET_HISTORY: break; case TI_TASK_DEL_HISTORY: break; - case TI_TASK_COMMIT_ADD: return ctask__commit_add(thing, up); + case TI_TASK_COMMIT: return ctask__commit(thing, up); } log_critical("unknown collection task: %"PRIu64, mp_task.via.u64); diff --git a/src/ti/task.c b/src/ti/task.c index f17cc807..ea70dff6 100644 --- a/src/ti/task.c +++ b/src/ti/task.c @@ -3159,7 +3159,7 @@ int ti_task_add_commit_add(ti_task_t * task, ti_commit_t * commit) msgpack_pack_array(&pk, 2); - msgpack_pack_uint8(&pk, TI_TASK_COMMIT_ADD); + msgpack_pack_uint8(&pk, TI_TASK_COMMIT); (void) ti_commit_to_pk(commit, &pk); data = (ti_data_t *) buffer.data; diff --git a/src/ti/ttask.c b/src/ti/ttask.c index a3fbc8a6..d320c065 100644 --- a/src/ti/ttask.c +++ b/src/ti/ttask.c @@ -802,7 +802,7 @@ static int ttask__del_history(mp_unp_t * up) return 0; } -static int ttask__commit_add(mp_unp_t * up) +static int ttask__commit(mp_unp_t * up) { vec_t ** commits = &ti.commits; ti_commit_t * commit = ti_commit_from_up(up); @@ -1863,7 +1863,7 @@ int ti_ttask_run(ti_change_t * change, mp_unp_t * up) case TI_TASK_WHITELIST_DEL: return ttask__whitelist_del(up); case TI_TASK_SET_HISTORY: return ttask__set_history(up); case TI_TASK_DEL_HISTORY: return ttask__del_history(up); - case TI_TASK_COMMIT_ADD: return ttask__commit_add(up); + case TI_TASK_COMMIT: return ttask__commit(up); } log_critical("unknown thingsdb task: %"PRIu64, mp_task.via.u64); From 005734863b3dd3a43c450f6fb6b72b6368b5d41f Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 5 Nov 2025 20:59:20 +0100 Subject: [PATCH 23/23] outline --- inc/ti.h | 2 +- inc/ti/collection.t.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inc/ti.h b/inc/ti.h index a80fffbe..aa0bfdbf 100644 --- a/inc/ti.h +++ b/inc/ti.h @@ -107,7 +107,7 @@ struct ti_s vec_t * users; /* ti_user_t */ vec_t * access_node; /* ti_auth_t */ vec_t * access_thingsdb; /* ti_auth_t */ - vec_t * commits; /* ti_commit_t */ + vec_t * commits; /* ti_commit_t */ smap_t * procedures; /* ti_procedure_t */ smap_t * names; /* weak map for ti_name_t */ smap_t * qcache; /* pointer to cache in stack */ diff --git a/inc/ti/collection.t.h b/inc/ti/collection.t.h index cb1769ab..994d5b54 100644 --- a/inc/ti/collection.t.h +++ b/inc/ti/collection.t.h @@ -40,7 +40,7 @@ struct ti_collection_s uv_mutex_t * lock; /* only for watch/ unwatch/ away-mode */ vec_t * futures; /* no reference, type: ti_future_t */ vec_t * vtasks; /* tasks, type: ti_vtask_t */ - vec_t * commits; /* migration changes ti_commit_t */ + vec_t * commits; /* migration changes ti_commit_t */ guid_t guid; /* derived from collection->id */ };