From 55e47be1f766b5cb705ff86ac0cb3e7ed768b207 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:21:01 -0600 Subject: [PATCH 01/41] add clean task --- config.nims | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/config.nims b/config.nims index 189a8fb..76ba5bf 100644 --- a/config.nims +++ b/config.nims @@ -3,19 +3,24 @@ switch("nimcache", ".nimcache") task buildimporter, "Build ormin_importer": exec "nim c -o:./tools/ormin_importer tools/ormin_importer" -task test, "Run all test suite": - buildimporterTask() +task clean, "Clean generated files": rmFile("tests/forum_model_sqlite.nim") rmFile("tests/model_sqlite.nim") + rmFile("tests/forum_model_postgres.nim") + rmFile("tests/model_postgre.nim") + +task test, "Run all test suite": + buildimporterTask() + cleanTask() + exec "nim c -f -r tests/tfeature" exec "nim c -f -r tests/tcommon" exec "nim c -f -r tests/tsqlite" task test_postgres, "Run PostgreSQL test suite": # Skip PostgreSQL tests as they require a running PostgreSQL server - rmFile("tests/forum_model_postgres.nim") - rmFile("tests/model_postgre.nim") + cleanTask() exec "nim c -r -d:postgre tests/tfeature" exec "nim c -r -d:postgre tests/tcommon" From 072f949d47671fb241e47632e6cf3703889e5f03 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:31:37 -0600 Subject: [PATCH 02/41] add composite key test --- tests/model_sqlite.sql | 16 ++++++++++++- tests/tcommon.nim | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/model_sqlite.sql b/tests/model_sqlite.sql index 2c8a41d..a966917 100644 --- a/tests/model_sqlite.sql +++ b/tests/model_sqlite.sql @@ -22,4 +22,18 @@ create table if not exists tb_timestamp( create table if not exists tb_json( typjson json not null -); \ No newline at end of file +); + +create table if not exists tb_composite_pk( + pk1 integer not null, + pk2 integer not null, + message text not null, + primary key (pk1, pk2) +); + +create table if not exists tb_composite_fk( + id integer not null, + fk1 integer not null, + fk2 integer not null, + foreign key (fk1, fk2) references tb_composite_pk(pk1, pk2) +); diff --git a/tests/tcommon.nim b/tests/tcommon.nim index 6df6db6..b6c30d9 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -338,3 +338,55 @@ suite "json": select tb_json(typjson) produce json check res == %*js.mapIt(%*{"typjson": it}) + + +let cps = [ + (1, 1, "one-one"), + (1, 2, "one-two"), + (2, 1, "two-one") +] + +suite "composite_pk_insert": + setup: + db.dropTable(sqlFile, "tb_composite_pk") + db.createTable(sqlFile, "tb_composite_pk") + + test "insert": + for r in cps: + query: + insert tb_composite_pk(pk1 = ?r[0], pk2 = ?r[1], message = ?r[2]) + check db.getValue(sql"select count(*) from tb_composite_pk") == $cps.len + + test "json": + for r in cps: + let v = %*{"pk1": r[0], "pk2": r[1], "message": r[2]} + query: + insert tb_composite_pk(pk1 = %v["pk1"], pk2 = %v["pk2"], message = %v["message"]) + check db.getValue(sql"select count(*) from tb_composite_pk") == $cps.len + + +suite "composite_pk": + db.dropTable(sqlFile, "tb_composite_pk") + db.createTable(sqlFile, "tb_composite_pk") + + let insertSql = sql"insert into tb_composite_pk(pk1, pk2, message) values (?, ?, ?)" + for r in cps: + db.exec(insertSql, r[0], r[1], r[2]) + doAssert db.getValue(sql"select count(*) from tb_composite_pk") == $cps.len + + test "query": + let res = query: + select tb_composite_pk(pk1, pk2, message) + check res == cps + + test "where": + let res = query: + select tb_composite_pk(pk1, pk2, message) + where pk1 == ?cps[0][0] + check res == cps.filterIt(it[0] == cps[0][0]) + + test "json": + let res = query: + select tb_composite_pk(pk1, pk2, message) + produce json + check res == %*cps.mapIt(%*{"pk1": it[0], "pk2": it[1], "message": it[2]}) From 520ec8217e10f49b4d5f76a5a46f238e6cb6d4aa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:51:29 -0600 Subject: [PATCH 03/41] add postgres setup scripts --- config.nims | 9 ++++++-- tests/model_postgre.sql | 23 ++++++++++++++++----- tools/ormin_importer.nim | 39 ++++++++++++++++++++++++++++------- tools/setup_postgres.sh | 33 +++++++++++++++++++++++++++++ tools/setup_postgres.sql | 21 +++++++++++++++++++ tools/setup_postgres_role.sql | 9 ++++++++ 6 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 tools/setup_postgres.sh create mode 100644 tools/setup_postgres.sql create mode 100644 tools/setup_postgres_role.sql diff --git a/config.nims b/config.nims index 76ba5bf..70ed511 100644 --- a/config.nims +++ b/config.nims @@ -18,9 +18,14 @@ task test, "Run all test suite": exec "nim c -f -r tests/tcommon" exec "nim c -f -r tests/tsqlite" +task setup_postgres, "Ensure local Postgres has test DB/user": + # Use a simple script to avoid Nim/psql quoting pitfalls + exec "bash -lc 'bash tools/setup_postgres.sh'" + task test_postgres, "Run PostgreSQL test suite": - # Skip PostgreSQL tests as they require a running PostgreSQL server cleanTask() + buildimporterTask() + setup_postgresTask() exec "nim c -r -d:postgre tests/tfeature" exec "nim c -r -d:postgre tests/tcommon" @@ -32,4 +37,4 @@ task buildexamples, "Build examples: chat and forum": selfExec "js examples/chat/frontend" selfExec "c examples/forum/forum" selfExec "c examples/forum/forumproto" - selfExec "c examples/tweeter/src/tweeter" \ No newline at end of file + selfExec "c examples/tweeter/src/tweeter" diff --git a/tests/model_postgre.sql b/tests/model_postgre.sql index 52aabe5..03ab382 100644 --- a/tests/model_postgre.sql +++ b/tests/model_postgre.sql @@ -12,15 +12,28 @@ create table if not exists tb_float( ); create table if not exists tb_string( - typstring varchar not null + typstring text not null ); create table if not exists tb_timestamp( - dt timestamp not null, - dtn timestamptz not null, - dtz timestamptz not null + dt1 timestamp not null, + dt2 timestamp not null ); create table if not exists tb_json( typjson json not null -); \ No newline at end of file +); + +create table if not exists tb_composite_pk( + pk1 integer not null, + pk2 integer not null, + message text not null, + primary key (pk1, pk2) +); + +create table if not exists tb_composite_fk( + id integer not null, + fk1 integer not null, + fk2 integer not null, + foreign key (fk1, fk2) references tb_composite_pk(pk1, pk2) +); diff --git a/tools/ormin_importer.nim b/tools/ormin_importer.nim index 38c402c..53ca518 100644 --- a/tools/ormin_importer.nim +++ b/tools/ormin_importer.nim @@ -125,17 +125,40 @@ proc collectTables(n: SqlNode; t: var KnownTables) = typ: typ, primaryKey: hasAttribute(it, {nkPrimaryKey}), refs: hasRefs(it)) + # Handle table-level foreign keys, including composite FKs for i in 1../dev/null 2>&1; then + psql -v ON_ERROR_STOP=1 -U postgres -d "$db" -f "$file" "$@" + fi +} + +run_psql_cmd() { + local db="$1"; shift + local cmd="$1"; shift + if ! psql -v ON_ERROR_STOP=1 -d "$db" -c "$cmd" "$@" >/dev/null 2>&1; then + psql -v ON_ERROR_STOP=1 -U postgres -d "$db" -c "$cmd" "$@" + fi +} + +# Create role 'test' if needed +run_psql_file postgres tools/setup_postgres_role.sql + +# Create database 'test' if needed (must be outside DO block) +if ! psql -v ON_ERROR_STOP=1 -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'test'" | grep -q 1; then + run_psql_cmd postgres "CREATE DATABASE test OWNER test" +fi + +# Grant privileges on public schema +run_psql_cmd test "GRANT ALL PRIVILEGES ON SCHEMA public TO test" + +echo "Postgres test DB/user ensured (role 'test', db 'test')." + diff --git a/tools/setup_postgres.sql b/tools/setup_postgres.sql new file mode 100644 index 0000000..b0dc3c8 --- /dev/null +++ b/tools/setup_postgres.sql @@ -0,0 +1,21 @@ +DO +$$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test') THEN + CREATE ROLE test LOGIN PASSWORD 'test'; + END IF; +END +$$; + +DO +$$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_database WHERE datname = 'test') THEN + CREATE DATABASE test OWNER test; + END IF; +END +$$; + +\connect test +GRANT ALL PRIVILEGES ON SCHEMA public TO test; + diff --git a/tools/setup_postgres_role.sql b/tools/setup_postgres_role.sql new file mode 100644 index 0000000..6a19451 --- /dev/null +++ b/tools/setup_postgres_role.sql @@ -0,0 +1,9 @@ +DO +$$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'test') THEN + CREATE ROLE test LOGIN PASSWORD 'test'; + END IF; +END +$$; + From eff70fe55e46129c4754e0694eaf5a4ff2217589 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 22:54:00 -0600 Subject: [PATCH 04/41] add postgres setup scripts --- tests/tcommon.nim | 2 +- tests/tfeature.nim | 2 +- tests/tpostgre.nim | 2 +- tools/setup_postgres.sh | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/tcommon.nim b/tests/tcommon.nim index b6c30d9..fd94a84 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -8,7 +8,7 @@ when defined(postgre): const backend = DbBackend.postgre importModel(backend, "model_postgre") const sqlFileName = "model_postgre.sql" - let db {.global.} = open("localhost", "test", "test", "test") + let db {.global.} = open("localhost", "test", "test", "test_ormin") else: from db_connector/db_sqlite import exec, getValue diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 7849d85..36e5b39 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -13,7 +13,7 @@ when defined postgre: const backend = DbBackend.postgre importModel(backend, "forum_model_postgres") const sqlFileName = "forum_model_postgres.sql" - let db {.global.} = open("localhost", "test", "test", "test") + let db {.global.} = open("localhost", "test", "test", "test_ormin") else: from db_connector/db_sqlite import exec, getValue diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index ff7db69..bb19e96 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -8,7 +8,7 @@ import ./utils importModel(DbBackend.postgre, "model_postgre") let - db {.global.} = open("localhost", "test", "test", "test") + db {.global.} = open("localhost", "test", "test", "test_ormin") testDir = currentSourcePath.parentDir() sqlFile = testDir / "model_postgre.sql" diff --git a/tools/setup_postgres.sh b/tools/setup_postgres.sh index 9f6e3d4..ff7fff3 100644 --- a/tools/setup_postgres.sh +++ b/tools/setup_postgres.sh @@ -22,12 +22,12 @@ run_psql_cmd() { run_psql_file postgres tools/setup_postgres_role.sql # Create database 'test' if needed (must be outside DO block) -if ! psql -v ON_ERROR_STOP=1 -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'test'" | grep -q 1; then - run_psql_cmd postgres "CREATE DATABASE test OWNER test" +if ! psql -v ON_ERROR_STOP=1 -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'test_ormin'" | grep -q 1; then + run_psql_cmd postgres "CREATE DATABASE test_ormin OWNER test" fi # Grant privileges on public schema -run_psql_cmd test "GRANT ALL PRIVILEGES ON SCHEMA public TO test" +run_psql_cmd test_ormin "GRANT ALL PRIVILEGES ON SCHEMA public TO test" -echo "Postgres test DB/user ensured (role 'test', db 'test')." +echo "Postgres test DB/user ensured (role 'test', db 'test_ormin')." From 131c079d37fafa57d994342b3ecbe355cb248d76 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 23:09:20 -0600 Subject: [PATCH 05/41] fix postgres on macosx --- config.nims | 9 ++++++--- ormin/ormin_postgre.nim | 2 +- tests/tcommon.nim | 3 +++ tests/tfeature.nim | 3 +++ tests/tpostgre.nim | 4 ++++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/config.nims b/config.nims index 70ed511..7014d21 100644 --- a/config.nims +++ b/config.nims @@ -26,10 +26,13 @@ task test_postgres, "Run PostgreSQL test suite": cleanTask() buildimporterTask() setup_postgresTask() + # Pre-generate Postgres models to avoid include timing issues + exec "./tools/ormin_importer tests/forum_model_postgres.sql" + exec "./tools/ormin_importer tests/model_postgre.sql" - exec "nim c -r -d:postgre tests/tfeature" - exec "nim c -r -d:postgre tests/tcommon" - exec "nim c -r tests/tpostgre" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tfeature" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tcommon" + exec "nim c -f -d:nimDebugDlOpen -r tests/tpostgre" task buildexamples, "Build examples: chat and forum": buildimporterTask() diff --git a/ormin/ormin_postgre.nim b/ormin/ormin_postgre.nim index 7f5248c..527eb4e 100644 --- a/ormin/ormin_postgre.nim +++ b/ormin/ormin_postgre.nim @@ -219,7 +219,7 @@ template startQuery*(db: DbConn; s: PStmt) = template stopQuery*(db: DbConn; s: PStmt) = pqclear(queryResult) -template stepQuery*(db: DbConn; s: PStmt; returnsData: int): bool = +template stepQuery*(db: DbConn; s: PStmt; returnsData: bool): bool = inc queryI queryI < queryLen diff --git a/tests/tcommon.nim b/tests/tcommon.nim index fd94a84..9d361d8 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -3,6 +3,9 @@ import ormin import ormin/db_utils when defined(postgre): + when defined(macosx): + {.passL: " " & gorge("pkg-config --libs libpq").} + {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} from db_postgres import exec, getValue const backend = DbBackend.postgre diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 36e5b39..ee32101 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -8,6 +8,9 @@ when NimVersion < "1.2.0": import ./compat let testDir = currentSourcePath.parentDir() when defined postgre: + when defined(macosx): + {.passL: " " & gorge("pkg-config --libs libpq").} + {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} from db_connector/db_postgres import exec, getValue const backend = DbBackend.postgre diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index bb19e96..ac890d0 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -2,6 +2,10 @@ import unittest, json, strutils, macros, times, os, sequtils import db_connector/postgres import ormin +when defined(macosx): + {.passL: " " & gorge("pkg-config --libs libpq").} + {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} + from db_connector/postgres import exec, getValue import ./utils From 2d991594d485db44511cacf6d4f3716d821902aa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 23:39:58 -0600 Subject: [PATCH 06/41] fix postgres --- ormin/db_utils.nim | 6 +++--- ormin/ormin_postgre.nim | 2 +- ormin/queries.nim | 21 +++++++++++++-------- tests/forum_model_postgres.sql | 12 ++++++------ tests/model_postgre.sql | 5 +++-- tests/tcommon.nim | 5 ++--- tests/tfeature.nim | 1 - tests/tpostgre.nim | 22 +++++++++++----------- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index f424657..5446bad 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,6 +1,6 @@ import db_connector/db_common, strutils, strformat, re -from db_connector/db_postgres import nil -from db_connector/db_sqlite import nil +import db_connector/db_postgres as db_postgres +import db_connector/db_sqlite as db_sqlite type DbConn = db_postgres.DbConn | db_sqlite.DbConn @@ -35,4 +35,4 @@ proc dropTable*(db: DbConn; sqlFile, name: string) = when defined(postgre): db.exec(sql("drop table if exists " & n & " cascade")) else: - db.exec(sql("drop table if exists " & n)) \ No newline at end of file + db.exec(sql("drop table if exists " & n)) diff --git a/ormin/ormin_postgre.nim b/ormin/ormin_postgre.nim index 527eb4e..fce1ab9 100644 --- a/ormin/ormin_postgre.nim +++ b/ormin/ormin_postgre.nim @@ -212,7 +212,7 @@ template startQuery*(db: DbConn; s: PStmt) = nil, nil, nil, 0) if pqResultStatus(queryResult) == PGRES_COMMAND_OK: discard # insert does not returns data in pg - elif pqResultStatus(queryResult) != PGRES_TUPLES_OK: dbError(db) + elif pqResultStatus(queryResult) != PGRES_TUPLES_OK: ormin_postgre.dbError(db) var queryI {.inject.} = cint(-1) var queryLen {.inject.} = pqntuples(queryResult) diff --git a/ormin/queries.nim b/ormin/queries.nim index 12c7c7b..6a3577e 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -143,11 +143,12 @@ proc checkBool(a: DbType; n: NimNode) = macros.error "expected type 'bool', but got: " & $a, n proc checkInt(a: DbType; n: NimNode) = - if a.kind != dbInt: + if a.kind notin {dbInt, dbSerial}: macros.error "expected type 'int', but got: " & $a, n proc checkCompatible(a, b: DbType; n: NimNode) = - if a.kind != b.kind: + # Treat serial and int as compatible + if not (a.kind == b.kind or (a.kind == dbSerial and b.kind == dbInt) or (a.kind == dbInt and b.kind == dbSerial)): macros.error "incompatible types: " & $a & " and " & $b, n proc checkCompatibleSet(a, b: DbType; n: NimNode) = @@ -454,6 +455,8 @@ proc generateRoutine(name: NimNode, q: QueryBuilder; let prepare = newVarStmt(prepStmt, newCall(bindSym"prepareStmt", ident"db", newLit(sql))) let body = newStmtList() + # Ensure the prepared statement is created before binding/starting the query + body.add prepare var finalParams = newNimNode(nnkFormalParams) if q.retTypeIsJson: @@ -471,7 +474,12 @@ proc generateRoutine(name: NimNode, q: QueryBuilder; if k != nnkIteratorDef: rtyp = nnkBracketExpr.newTree(ident"seq", rtyp) finalParams.add rtyp - finalParams.add newIdentDefs(ident"db", ident("DbConn")) + when dbBackend == DbBackend.postgre: + finalParams.add newIdentDefs(ident"db", newTree(nnkDotExpr, ident"ormin_postgre", ident"DbConn")) + elif dbBackend == DbBackend.sqlite: + finalParams.add newIdentDefs(ident"db", newTree(nnkDotExpr, ident"ormin_sqlite", ident"DbConn")) + else: + finalParams.add newIdentDefs(ident"db", ident("DbConn")) var i = 1 if q.params.len > 0: body.add newCall(bindSym"startBindings", prepStmt, newLit(q.params.len)) @@ -869,14 +877,11 @@ proc queryImpl(q: QueryBuilder; body: NimNode; attempt, produceJson: bool): NimN if b.kind == nnkCommand: queryh(b, q) else: macros.error "illformed query", b let sql = queryAsString(q, body) - let prepStmt = genSym(nskVar) + let prepStmt = genSym(nskLet) let res = genSym(nskVar) - let prepStmtCall = newCall(bindSym"prepareStmt", ident"db", newLit sql) result = newTree( if q.retType.len > 0: nnkStmtListExpr else: nnkStmtList, - # really hack-ish - newGlobalVar(prepStmt, newCall(bindSym"typeof", prepStmtCall), newEmptyNode()), - getAst(once(newAssignment(prepStmt, prepStmtCall))) + newLetStmt(prepStmt, newCall(bindSym"prepareStmt", ident"db", newLit sql)) ) let rtyp = if q.retType.len > 1 or q.retType.len == 0: q.retType diff --git a/tests/forum_model_postgres.sql b/tests/forum_model_postgres.sql index 8ed5457..8ce68b0 100644 --- a/tests/forum_model_postgres.sql +++ b/tests/forum_model_postgres.sql @@ -1,5 +1,5 @@ create table if not exists thread( - id integer primary key, + id serial primary key, name varchar(100) not null, views integer not null, modified timestamp not null default CURRENT_TIMESTAMP @@ -8,7 +8,7 @@ create table if not exists thread( create unique index if not exists ThreadNameIx on thread (name); create table if not exists person( - id integer primary key, + id serial primary key, name varchar(20) not null, password varchar(32) not null, email varchar(30) not null, @@ -22,7 +22,7 @@ create table if not exists person( create unique index if not exists UserNameIx on person (name); create table if not exists post( - id integer primary key, + id serial primary key, author integer not null, ip varchar(20) not null, header varchar(100) not null, @@ -35,7 +35,7 @@ create table if not exists post( ); create table if not exists session( - id integer primary key, + id serial primary key, ip varchar(20) not null, password varchar(32) not null, userid integer not null, @@ -44,7 +44,7 @@ create table if not exists session( ); create table if not exists antibot( - id integer primary key, + id serial primary key, ip varchar(20) not null, answer varchar(30) not null, created timestamp not null default CURRENT_TIMESTAMP @@ -57,4 +57,4 @@ create table if not exists error( ); create index PersonStatusIdx on person(status); -create index PostByAuthorIdx on post(thread, author); \ No newline at end of file +create index PostByAuthorIdx on post(thread, author); diff --git a/tests/model_postgre.sql b/tests/model_postgre.sql index 03ab382..ac6c4e9 100644 --- a/tests/model_postgre.sql +++ b/tests/model_postgre.sql @@ -16,8 +16,9 @@ create table if not exists tb_string( ); create table if not exists tb_timestamp( - dt1 timestamp not null, - dt2 timestamp not null + dt timestamp not null, + dtn timestamptz not null, + dtz timestamptz not null ); create table if not exists tb_json( diff --git a/tests/tcommon.nim b/tests/tcommon.nim index 9d361d8..e407516 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -4,14 +4,13 @@ import ormin/db_utils when defined(postgre): when defined(macosx): - {.passL: " " & gorge("pkg-config --libs libpq").} {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} - from db_postgres import exec, getValue + import db_connector/db_postgres as db_postgres const backend = DbBackend.postgre importModel(backend, "model_postgre") const sqlFileName = "model_postgre.sql" - let db {.global.} = open("localhost", "test", "test", "test_ormin") + let db {.global.} = db_postgres.open("localhost", "test", "test", "test_ormin") else: from db_connector/db_sqlite import exec, getValue diff --git a/tests/tfeature.nim b/tests/tfeature.nim index ee32101..798d3f3 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -9,7 +9,6 @@ let testDir = currentSourcePath.parentDir() when defined postgre: when defined(macosx): - {.passL: " " & gorge("pkg-config --libs libpq").} {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} from db_connector/db_postgres import exec, getValue diff --git a/tests/tpostgre.nim b/tests/tpostgre.nim index ac890d0..c9b63cf 100644 --- a/tests/tpostgre.nim +++ b/tests/tpostgre.nim @@ -1,18 +1,18 @@ import unittest, json, strutils, macros, times, os, sequtils -import db_connector/postgres +# Postgres connection handled through ormin_postgre backend import ormin +import ormin/ormin_postgre as ormin_postgre +import db_connector/db_postgres as db_postgres +import ormin/db_utils when defined(macosx): - {.passL: " " & gorge("pkg-config --libs libpq").} {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} -from db_connector/postgres import exec, getValue -import ./utils importModel(DbBackend.postgre, "model_postgre") let - db {.global.} = open("localhost", "test", "test", "test_ormin") + db {.global.} = ormin_postgre.open("localhost", "test", "test", "test_ormin") testDir = currentSourcePath.parentDir() sqlFile = testDir / "model_postgre.sql" @@ -59,23 +59,23 @@ suite "timestamp_insert": test "insert": query: insert tb_timestamp(dt = ?dt1, dtn = ?dtn1, dtz = ?dtz1) - check db.getValue(sql"select count(*) from tb_timestamp") == "1" + check db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "1" test "json": query: insert tb_timestamp(dt = %dtjson1["dt"], dtn = %dtjson1["dtn"], dtz = %dtjson1["dtz"]) - check db.getValue(sql"select count(*) from tb_timestamp") == "1" + check db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "1" suite "timestamp": db.dropTable(sqlFile, "tb_timestamp") db.createTable(sqlFile, "tb_timestamp") - db.exec(insertSql, dtStr1, dtnStr1, dtzStr1) - db.exec(insertSql, dtStr2, dtnStr2, dtzStr2) - db.exec(insertSql, dtStr3, dtnStr3, dtzStr3) - doAssert db.getValue(sql"select count(*) from tb_timestamp") == "3" + db_postgres.exec(db, insertSql, dtStr1, dtnStr1, dtzStr1) + db_postgres.exec(db, insertSql, dtStr2, dtnStr2, dtzStr2) + db_postgres.exec(db, insertSql, dtStr3, dtnStr3, dtzStr3) + doAssert db_postgres.getValue(db, sql"select count(*) from tb_timestamp") == "3" test "query": let res = query: From d5707d097cf83bf8ab0c59bdb310a967df89ce44 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 10 Sep 2025 23:55:46 -0600 Subject: [PATCH 07/41] fix postgres --- tests/tfeature.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 798d3f3..2641073 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -529,7 +529,7 @@ suite "query": test "insert_return_answer": # test returning non-id parameter - let expectedanswer = "just insert" + let expectedanswer = "just insert another" let answer = query: insert antibot(id = 9, ip = "", answer = ?expectedanswer) returning answer @@ -538,7 +538,7 @@ suite "query": test "insert_return_id_auto": # test returning id column let answer = query: - insert antibot(ip = "", answer = "just another insert") + insert antibot(ip = "", answer = "just auto insert") returning id check answer == 10 From 62669fb767e543b9bb7a532aacaa846c97a3960a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:17:46 -0600 Subject: [PATCH 08/41] fix postgres test, document using db_connector --- README.md | 8 ++++++++ tests/tfeature.nim | 3 +++ 2 files changed, 11 insertions(+) diff --git a/README.md b/README.md index cac5a52..01cd9e0 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,14 @@ for row in db.postsIter(userId): Both forms accept parameters matching the `?`/`%` placeholders and produce the same return types as an inline `query` block. +## Running Arbitrary SQL + +The standard `db_connector` APIs can be imported and used. For example: + +```nim +discard db.getValue(sql"select setval('antibot_id_seq', 10, false)") +``` + ## Additional Facilities - **Protocol DSL** – The `protocol` macro lets you describe paired server/client handlers that communicate via JSON messages. Sections use keywords like `recv`, `broadcast` and `send`, and every server block must be mirrored by a client block. The chat example demonstrates this code generation. diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 2641073..31eaf8a 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -537,6 +537,9 @@ suite "query": test "insert_return_id_auto": # test returning id column + when defined(postgre): + # fix postgres sequence so next nextval returns 10 + discard db.getValue(sql"select setval('antibot_id_seq', ?, ?)", 10, false) let answer = query: insert antibot(ip = "", answer = "just auto insert") returning id From 764035ec02e8c5074f1799e0d92d1a441ceee5ba Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:18:10 -0600 Subject: [PATCH 09/41] fix postgres test, document using db_connector --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec23b13..7d21850 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: postgresql version: '14' postgresql user: 'test' postgresql password: 'test' - postgresql db: 'test' + postgresql db: 'test_ormin' - name: Install system dependencies run: | From 41beb00c0312a6f40ce9a59204f5a3b94620c748 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:19:33 -0600 Subject: [PATCH 10/41] run pg tests --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d21850..e7d0662 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,8 +52,11 @@ jobs: nimble --useSystemNim install karax -y nimble --useSystemNim install jester -y - - name: Run test + - name: Run test sqlite run: nimble --useSystemNim test + + - name: Run test postgres + run: nimble --useSystemNim test_postgres env: PGPASSWORD: test From 5f69404672011bff7c58395af22d65956c80d93d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:28:06 -0600 Subject: [PATCH 11/41] run pg tests --- .github/workflows/test.yml | 1 + ormin.nimble | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e7d0662..96f3e83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,7 @@ jobs: nimble --useSystemNim install -d nimble --useSystemNim install karax -y nimble --useSystemNim install jester -y + nimble --useSystemNim install websocket -y - name: Run test sqlite run: nimble --useSystemNim test diff --git a/ormin.nimble b/ormin.nimble index 23e4d37..895cac6 100644 --- a/ormin.nimble +++ b/ormin.nimble @@ -12,11 +12,9 @@ installExt = @["nim"] requires "nim >= 2.0.0" requires "db_connector >= 0.1.0" -feature "websocket": +feature "examples": requires "websocket >= 0.2.2" -feature "karax": requires "karax" -feature "jester": requires "jester" include "config.nims" From 1db14481ceb7b40da253a789b7283fbe960a0247 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:34:00 -0600 Subject: [PATCH 12/41] run pg tests --- .github/workflows/test.yml | 12 +++++------- config.nims | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 96f3e83..5909efe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,13 +33,11 @@ jobs: with: nim-version: ${{ matrix.nim }} - - name: Setup postgresql - uses: harmon758/postgresql-action@v1 - with: - postgresql version: '14' - postgresql user: 'test' - postgresql password: 'test' - postgresql db: 'test_ormin' + - name: Start PostgreSQL + run: | + sudo systemctl start postgresql@14-main + sudo -u postgres psql -c "CREATE USER test WITH SUPERUSER CREATEDB CREATEROLE PASSWORD 'test';" + sudo -u postgres psql -c "CREATE DATABASE test_ormin OWNER test;" - name: Install system dependencies run: | diff --git a/config.nims b/config.nims index 7014d21..a567662 100644 --- a/config.nims +++ b/config.nims @@ -25,7 +25,7 @@ task setup_postgres, "Ensure local Postgres has test DB/user": task test_postgres, "Run PostgreSQL test suite": cleanTask() buildimporterTask() - setup_postgresTask() + # setup_postgresTask() # Pre-generate Postgres models to avoid include timing issues exec "./tools/ormin_importer tests/forum_model_postgres.sql" exec "./tools/ormin_importer tests/model_postgre.sql" From 446b1b6a8c2fbc5e15888ed426404bce6364f25e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:38:29 -0600 Subject: [PATCH 13/41] run pg tests --- .github/workflows/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5909efe..67b33c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,22 @@ on: [push] jobs: build: runs-on: ubuntu-latest + + services: + postgres: + image: postgres:14 + env: + POSTGRES_PASSWORD: test + POSTGRES_USER: test + POSTGRES_DB: test_ormin + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + strategy: matrix: nim: From 30bf3e4a405e22ea6f4cf25e12fcc84981c81368 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:39:17 -0600 Subject: [PATCH 14/41] run pg tests --- .github/workflows/test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67b33c6..1ce6367 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,12 +49,6 @@ jobs: with: nim-version: ${{ matrix.nim }} - - name: Start PostgreSQL - run: | - sudo systemctl start postgresql@14-main - sudo -u postgres psql -c "CREATE USER test WITH SUPERUSER CREATEDB CREATEROLE PASSWORD 'test';" - sudo -u postgres psql -c "CREATE DATABASE test_ormin OWNER test;" - - name: Install system dependencies run: | sudo apt-get update From 55b8724006a7762a42d5e64bf131d1954f5cd939 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 00:44:29 -0600 Subject: [PATCH 15/41] run pg tests --- .github/workflows/test.yml | 5 ----- ormin/db_utils.nim | 11 +++++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ce6367..644034a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,11 +49,6 @@ jobs: with: nim-version: ${{ matrix.nim }} - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libpcre3-dev - - name: Install dependencies run: | nimble --useSystemNim install -d diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 5446bad..a56c2ec 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,4 +1,4 @@ -import db_connector/db_common, strutils, strformat, re +import db_connector/db_common, strutils, strformat, pegs import db_connector/db_postgres as db_postgres import db_connector/db_sqlite as db_sqlite @@ -6,10 +6,13 @@ type DbConn = db_postgres.DbConn | db_sqlite.DbConn iterator tablePairs(sqlFile: string): tuple[name, model: string] = let f = readFile(sqlFile) + let pat = peg"start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? { [A-Za-z_][A-Za-z0-9_]* }" for m in f.split(';'): - if m.strip() != "" and - m =~ re"\n*create\s+table(\s+if\s+not\s+exists)?\s+(\w+)": - yield (matches[1], m) + let s = m.toLowerAscii() + if m.strip() != "": + var caps = newSeq[string](1) + if s.match(pat, caps): + yield (caps[0], m) proc createTable*(db: DbConn; sqlFile: string) = for _, m in tablePairs(sqlFile): From 3bb38d806439eaf67f9dd07d05c6aba3651caf09 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 01:06:55 -0600 Subject: [PATCH 16/41] use pegs --- .gitignore | 1 + ormin.nimble | 2 +- ormin/db_utils.nim | 15 +++++++++++-- tests/db_utils_case_quoted.sql | 12 +++++++++++ tests/tdb_utils.nim | 39 ++++++++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/db_utils_case_quoted.sql create mode 100644 tests/tdb_utils.nim diff --git a/.gitignore b/.gitignore index d5f9ada..28353e0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ examples/tweeter/src/tweeterdeps/ deps/ nim.cfg .nimcache +tests/tdb_utils diff --git a/ormin.nimble b/ormin.nimble index 895cac6..9e582f6 100644 --- a/ormin.nimble +++ b/ormin.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.0" +version = "0.4.0" author = "Araq" description = "Prepared SQL statement generator. A lightweight ORM." license = "MIT" diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index a56c2ec..1e32efe 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -6,12 +6,23 @@ type DbConn = db_postgres.DbConn | db_sqlite.DbConn iterator tablePairs(sqlFile: string): tuple[name, model: string] = let f = readFile(sqlFile) - let pat = peg"start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? { [A-Za-z_][A-Za-z0-9_]* }" + let pat = peg""" + start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? ident + ident <- quoted / unquoted + quoted <- '"' { (!'"' .)+ } '"' + unquoted <- { [A-Za-z_][A-Za-z0-9_]* } + """ for m in f.split(';'): let s = m.toLowerAscii() if m.strip() != "": + var cleaned = newSeq[string]() + for ln in s.splitLines(): + let t = ln.strip() + if not t.startsWith("--"): + cleaned.add ln + let sc = cleaned.join("\n") var caps = newSeq[string](1) - if s.match(pat, caps): + if sc.match(pat, caps): yield (caps[0], m) proc createTable*(db: DbConn; sqlFile: string) = diff --git a/tests/db_utils_case_quoted.sql b/tests/db_utils_case_quoted.sql new file mode 100644 index 0000000..ef8db4e --- /dev/null +++ b/tests/db_utils_case_quoted.sql @@ -0,0 +1,12 @@ +-- lower case, upper case, and quoted table names + create table lower_table ( + id integer primary key + ); + + CREATE TABLE UPPER_TABLE ( + id integer primary key + ); + + create table "Quoted_Table" ( + id integer primary key + ); diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim new file mode 100644 index 0000000..3268b63 --- /dev/null +++ b/tests/tdb_utils.nim @@ -0,0 +1,39 @@ +import unittest, os +import db_connector/db_common +from db_connector/db_sqlite import open, exec, getValue +import ormin/db_utils + +let + db {.global.} = open(":memory:", "", "", "") + testDir = currentSourcePath.parentDir() + sqlFile = testDir / "db_utils_case_quoted.sql" + +let sqlContent = """ +-- lower case, upper case, and quoted table names + create table lower_table ( + id integer primary key + ); + + CREATE TABLE UPPER_TABLE ( + id integer primary key + ); + + create table "Quoted_Table" ( + id integer primary key + ); +""" + +writeFile(sqlFile, sqlContent) + +suite "db_utils: case and quoted names": + test "createTable creates all tables from SQL file": + db.createTable(sqlFile) + let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted_Table')") + check countAll == "3" + + test "createTable with specific lowercased name matches quoted": + # Use a new in-memory DB for isolation + let db2 = open(":memory:", "", "", "") + db2.createTable(sqlFile, "quoted_table") + let countQuoted = db2.getValue(sql"select count(*) from sqlite_master where type='table' and name = 'Quoted_Table'") + check countQuoted == "1" From 6587455289f2071a52ab98761b218879af9eac4d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 11 Sep 2025 01:08:56 -0600 Subject: [PATCH 17/41] use pegs --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 644034a..46ee496 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: CI -on: [push] +on: [push, pull_request] jobs: build: @@ -65,4 +65,4 @@ jobs: PGPASSWORD: test - name: Build examples - run: nimble buildexamples \ No newline at end of file + run: nimble buildexamples From 697341591dad0b28ec4835550d884481e7465393 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sat, 13 Sep 2025 06:20:52 -0600 Subject: [PATCH 18/41] Use parsesql for table name detection --- ormin/db_utils.nim | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 1e32efe..401852e 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -1,4 +1,5 @@ -import db_connector/db_common, strutils, strformat, pegs +import db_connector/db_common, strutils, strformat +import std/parsesql import db_connector/db_postgres as db_postgres import db_connector/db_sqlite as db_sqlite @@ -6,24 +7,18 @@ type DbConn = db_postgres.DbConn | db_sqlite.DbConn iterator tablePairs(sqlFile: string): tuple[name, model: string] = let f = readFile(sqlFile) - let pat = peg""" - start <- \s* 'create' \s+ 'table' \s+ ('if' \s+ 'not' \s+ 'exists' \s+)? ident - ident <- quoted / unquoted - quoted <- '"' { (!'"' .)+ } '"' - unquoted <- { [A-Za-z_][A-Za-z0-9_]* } - """ for m in f.split(';'): - let s = m.toLowerAscii() - if m.strip() != "": - var cleaned = newSeq[string]() - for ln in s.splitLines(): - let t = ln.strip() - if not t.startsWith("--"): - cleaned.add ln - let sc = cleaned.join("\n") - var caps = newSeq[string](1) - if sc.match(pat, caps): - yield (caps[0], m) + let stmt = m.strip() + if stmt.len == 0: continue + try: + let ast = parseSql(stmt) + if ast.len > 0: + let node = ast[0] + if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: + let tableName = node[0].strVal.toLowerAscii() + yield (tableName, stmt) + except SqlParseError: + discard proc createTable*(db: DbConn; sqlFile: string) = for _, m in tablePairs(sqlFile): From 9beb13d510acda145b0671c64aac2031b10f2281 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:04:05 -0600 Subject: [PATCH 19/41] updates --- ormin/db_utils.nim | 29 ++++++++++++++++------------- tests/tdb_utils.nim | 11 ++++++++++- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/ormin/db_utils.nim b/ormin/db_utils.nim index 401852e..e5f7a5c 100644 --- a/ormin/db_utils.nim +++ b/ormin/db_utils.nim @@ -5,20 +5,23 @@ import db_connector/db_sqlite as db_sqlite type DbConn = db_postgres.DbConn | db_sqlite.DbConn -iterator tablePairs(sqlFile: string): tuple[name, model: string] = +iterator tablePairs*(sqlFile: string): tuple[name, model: string] = + # Parse the entire SQL file and iterate statements via the SQL parser let f = readFile(sqlFile) - for m in f.split(';'): - let stmt = m.strip() - if stmt.len == 0: continue - try: - let ast = parseSql(stmt) - if ast.len > 0: - let node = ast[0] - if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: - let tableName = node[0].strVal.toLowerAscii() - yield (tableName, stmt) - except SqlParseError: - discard + let ast = parseSql(f) + if ast.len > 0: + # ast is a statement list; iterate each statement node + for i in 0 ..< ast.len: + let node = ast[i] + if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: + let tableName = node[0].strVal.toLowerAscii() + yield (tableName, $node) + else: + # Fallback: ast might be a single statement (not a list) + let node = ast + if node.kind in {nkCreateTable, nkCreateTableIfNotExists}: + let tableName = node[0].strVal.toLowerAscii() + yield (tableName, $node) proc createTable*(db: DbConn; sqlFile: string) = for _, m in tablePairs(sqlFile): diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 3268b63..78241b9 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -1,4 +1,4 @@ -import unittest, os +import unittest, os, sequtils import db_connector/db_common from db_connector/db_sqlite import open, exec, getValue import ormin/db_utils @@ -26,6 +26,15 @@ let sqlContent = """ writeFile(sqlFile, sqlContent) suite "db_utils: case and quoted names": + test "check tables names": + let pairs = tablePairs(sqlFile).toSeq() + check pairs.len == 3 + check pairs[0][0] == ("lower_table") + check pairs[1][0] == ("upper_table") + check pairs[2][0] == ("quoted_table") + + check pairs[0][1] == ("create table lower_table (id integer primary key)") + test "createTable creates all tables from SQL file": db.createTable(sqlFile) let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted_Table')") From 32e6594b1cadf03ef152d7c4a0bda75d4e9360e8 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:05:22 -0600 Subject: [PATCH 20/41] updates --- tests/tdb_utils.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 78241b9..1c01c86 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -33,7 +33,9 @@ suite "db_utils: case and quoted names": check pairs[1][0] == ("upper_table") check pairs[2][0] == ("quoted_table") - check pairs[0][1] == ("create table lower_table (id integer primary key)") + + echo pairs[0][1].repr() + check pairs[0][1] == ("create table lower_table (id integer primary key);") test "createTable creates all tables from SQL file": db.createTable(sqlFile) From 547d731f7e6074ef37ad568f2b99be4086873cc6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:08:51 -0600 Subject: [PATCH 21/41] updates --- tests/db_utils_case_quoted.sql | 2 +- tests/tdb_utils.nim | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/db_utils_case_quoted.sql b/tests/db_utils_case_quoted.sql index ef8db4e..d3c0121 100644 --- a/tests/db_utils_case_quoted.sql +++ b/tests/db_utils_case_quoted.sql @@ -7,6 +7,6 @@ id integer primary key ); - create table "Quoted_Table" ( + create table "Quoted Table" ( id integer primary key ); diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index 1c01c86..c3b6653 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -18,7 +18,7 @@ let sqlContent = """ id integer primary key ); - create table "Quoted_Table" ( + create table "Quoted Table" ( id integer primary key ); """ @@ -31,20 +31,21 @@ suite "db_utils: case and quoted names": check pairs.len == 3 check pairs[0][0] == ("lower_table") check pairs[1][0] == ("upper_table") - check pairs[2][0] == ("quoted_table") + check pairs[2][0] == ("quoted table") - echo pairs[0][1].repr() - check pairs[0][1] == ("create table lower_table (id integer primary key);") + check pairs[0][1] == "create table lower_table(id integer primary key );" + check pairs[1][1] == "create table UPPER_TABLE(id integer primary key );" + check pairs[2][1] == "create table \"Quoted Table\"(id integer primary key );" test "createTable creates all tables from SQL file": db.createTable(sqlFile) - let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted_Table')") + let countAll = db.getValue(sql"select count(*) from sqlite_master where type='table' and name in ('lower_table','UPPER_TABLE','Quoted Table')") check countAll == "3" test "createTable with specific lowercased name matches quoted": # Use a new in-memory DB for isolation let db2 = open(":memory:", "", "", "") - db2.createTable(sqlFile, "quoted_table") - let countQuoted = db2.getValue(sql"select count(*) from sqlite_master where type='table' and name = 'Quoted_Table'") + db2.createTable(sqlFile, "quoted table") + let countQuoted = db2.getValue(sql"select count(*) from sqlite_master where type='table' and name = 'Quoted Table'") check countQuoted == "1" From 236f72179d936a325e8dad5fd99046778f9efd57 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:10:21 -0600 Subject: [PATCH 22/41] updates --- tests/db_utils_case_quoted.sql | 6 ++++++ tests/tdb_utils.nim | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/db_utils_case_quoted.sql b/tests/db_utils_case_quoted.sql index d3c0121..cc25649 100644 --- a/tests/db_utils_case_quoted.sql +++ b/tests/db_utils_case_quoted.sql @@ -7,6 +7,12 @@ id integer primary key ); + -- std sql quoted table name create table "Quoted Table" ( id integer primary key ); + + -- sqlite quoted table name + create table `Quoted Table2` ( + id integer primary key + ); diff --git a/tests/tdb_utils.nim b/tests/tdb_utils.nim index c3b6653..435eaa4 100644 --- a/tests/tdb_utils.nim +++ b/tests/tdb_utils.nim @@ -18,9 +18,15 @@ let sqlContent = """ id integer primary key ); + -- std sql quoted table name create table "Quoted Table" ( id integer primary key ); + + -- sqlite quoted table name + create table `Quoted Table2` ( + id integer primary key + ); """ writeFile(sqlFile, sqlContent) @@ -28,10 +34,11 @@ writeFile(sqlFile, sqlContent) suite "db_utils: case and quoted names": test "check tables names": let pairs = tablePairs(sqlFile).toSeq() - check pairs.len == 3 + check pairs.len == 4 check pairs[0][0] == ("lower_table") check pairs[1][0] == ("upper_table") check pairs[2][0] == ("quoted table") + check pairs[3][0] == ("quoted table2") check pairs[0][1] == "create table lower_table(id integer primary key );" From 7cfe86a42e8e81deaf69c506a11065868c3c7050 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Sep 2025 07:25:24 -0600 Subject: [PATCH 23/41] adding transactions --- README.md | 35 ++++++- ormin/queries.nim | 227 ++++++++++++++++++++++++++++++++++++++++ tests/ttransactions.nim | 78 ++++++++++++++ 3 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 tests/ttransactions.nim diff --git a/README.md b/README.md index 01cd9e0..a00fbf1 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,40 @@ The tests include additional samples of JSON parameters and raw SQL expressions. ## Transactions and Batching -TODO! +Use `transaction:` to run multiple queries atomically. The block commits on success and rolls back on any exception. Nesting is supported via savepoints. `tryTransaction:` behaves the same but returns `bool` (false on database errors) without raising. + +Examples: + +```nim +# Commit on success +transaction: + query: + insert person(id = ?(1), name = ?"alice", password = ?"p", email = ?"a@x", salt = ?"s", status = ?"ok") + query: + update thread(views = views + 1) + where id == ?(42) + +# Rollback on error +let ok = tryTransaction: + query: + insert person(id = ?(2), name = ?"bob", password = ?"p", email = ?"b@x", salt = ?"s", status = ?"ok") + # Primary key violation => entire block is rolled back, ok = false + query: + insert person(id = ?(2), name = ?"duplicate", password = ?"p", email = ?"d@x", salt = ?"s", status = ?"x") + +# Nested transactions via savepoints +transaction: + query: + insert person(id = ?(3), name = ?"carol", password = ?"p", email = ?"c@x", salt = ?"s", status = ?"ok") + let innerOk = tryTransaction: + # This will fail and roll back to the savepoint + query: + insert person(id = ?(3), name = ?"duplicate", password = ?"p", email = ?"d@x", salt = ?"s", status = ?"x") + doAssert innerOk == false + # Continue outer transaction normally +``` + +PostgreSQL and SQLite are supported. The macros use `BEGIN/COMMIT/ROLLBACK` for the outermost transaction and `SAVEPOINT/RELEASE/ROLLBACK TO` for nested scopes. ## Reusable Procedures and Iterators diff --git a/ormin/queries.nim b/ormin/queries.nim index 6a3577e..a36acab 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -65,6 +65,26 @@ type # For SQLite: expression to return instead of last_insert_rowid() retExpr: NimNode +# Transaction state for nested transactions +var txDepth* {.threadvar.}: int + +# Execute a non-row SQL statement strictly (errors on failure) +template execNoRowsStrict*(sqlStmt: string) = + let s {.gensym.} = prepareStmt(db, sqlStmt) + startQuery(db, s) + if stepQuery(db, s, false): + stopQuery(db, s) + else: + stopQuery(db, s) + dbError(db) + +# Execute a non-row SQL statement, relying on startQuery to raise on failure +template execNoRowsLoose*(sqlStmt: string) = + let s {.gensym.} = prepareStmt(db, sqlStmt) + startQuery(db, s) + discard stepQuery(db, s, false) + stopQuery(db, s) + proc newQueryBuilder(): QueryBuilder {.compileTime.} = QueryBuilder(head: "", fromm: "", join: "", values: "", where: "", groupby: "", having: "", orderby: "", limit: "", offset: "", @@ -1011,6 +1031,213 @@ macro tryQuery*(body: untyped): untyped = when defined(debugOrminDsl): macros.hint("Ormin Query: " & repr(result), body) +# ------------------------- +# Transactions DSL +# ------------------------- + +macro transaction*(body: untyped): untyped = + ## Runs the body inside a database transaction. Commits on success, + ## rolls back on any exception and rethrows. + ## Supports nesting via savepoints. + let topSym = genSym(nskVar, "orminTop") + let spSym = genSym(nskLet, "orminSp") + let rvSym = genSym(nskLet, "orminRv") + result = newStmtList() + # block to ensure proper expression value propagation + var blockStmts = newStmtList() + blockStmts.add newVarStmt(topSym, newTree(nnkInfix, ident"==", ident"txDepth", newLit 0)) + blockStmts.add newCall(ident"inc", ident"txDepth") + blockStmts.add newLetStmt(spSym, newTree(nnkInfix, ident"&", newLit("ormin_tx_"), newCall(ident"$", ident"txDepth"))) + + # BEGIN or SAVEPOINT + var beginStmt = newStmtList() + beginStmt.add newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("begin")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("begin")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) + ) + ) + ) + + # body result + blockStmts.add newLetStmt(rvSym, newTree(nnkBlockStmt, newEmptyNode(), body)) + + # COMMIT or RELEASE SAVEPOINT + var commitStmt = newStmtList() + commitStmt.add newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("commit")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("commit")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) + ) + ) + ) + + # try/except wrapping + let tryBody = newStmtList(beginStmt, newEmptyNode()) + # after begin, execute; above rv already holds value + tryBody.add commitStmt + + let exceptDb = newTree(nnkExceptBranch, newTree(nnkRefTy, ident"DbError"), + newStmtList( + newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ) + ), + newTree(nnkRaiseStmt, newEmptyNode()) + ) + ) + let exceptAny = newTree(nnkExceptBranch, ident"Exception", + newStmtList( + newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ) + ), + newTree(nnkRaiseStmt, newEmptyNode()) + ) + ) + let finallyStmtTx = newTree(nnkFinally, newStmtList(newCall(ident"dec", ident"txDepth"))) + let tryStmt = newTree(nnkTryStmt, tryBody, exceptDb, exceptAny, finallyStmtTx) + blockStmts.add tryStmt + blockStmts.add rvSym + result = newTree(nnkStmtListExpr, newTree(nnkBlockStmt, newEmptyNode(), blockStmts)) + +macro tryTransaction*(body: untyped): untyped = + ## Same as `transaction` but returns bool and swallows DbError. + let topSym = genSym(nskVar, "orminTop") + let spSym = genSym(nskLet, "orminSp") + let okSym = genSym(nskVar, "orminOk") + result = newStmtList() + var blockStmts = newStmtList() + blockStmts.add newVarStmt(topSym, newTree(nnkInfix, ident"==", ident"txDepth", newLit 0)) + blockStmts.add newCall(ident"inc", ident"txDepth") + blockStmts.add newLetStmt(spSym, newTree(nnkInfix, ident"&", newLit("ormin_tx_"), newCall(ident"$", ident"txDepth"))) + blockStmts.add newTree(nnkVarSection, newIdentDefs(okSym, ident"bool", newLit(true))) + + # BEGIN or SAVEPOINT + var beginStmt = newStmtList() + beginStmt.add newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("begin")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("begin")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) + ) + ) + ) + + # Body execution as statements; ignore any result + let bodyBlk = newTree(nnkBlockStmt, newEmptyNode(), body) + + # COMMIT or RELEASE SAVEPOINT + var commitStmt = newStmtList() + commitStmt.add newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("commit")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("commit")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) + ) + ) + ) + + let tryBody = newStmtList(beginStmt, bodyBlk, commitStmt) + + let exceptDb = newTree(nnkExceptBranch, ident"DbError", + newStmtList( + newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ) + ), + newAssignment(okSym, newLit(false)) + ) + ) + let exceptAny = newTree(nnkExceptBranch, ident"Exception", + newStmtList( + newTree(nnkWhenStmt, + newTree(nnkElifBranch, + newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ), + newTree(nnkElse, + newTree(nnkIfStmt, + newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), + newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) + ) + ) + ), + newTree(nnkRaiseStmt, newEmptyNode()) + ) + ) + let finallyStmt = newTree(nnkFinally, newStmtList(newCall(ident"dec", ident"txDepth"))) + let tryStmt = newTree(nnkTryStmt, tryBody, exceptDb, exceptAny, finallyStmt) + blockStmts.add tryStmt + blockStmts.add okSym + result = newTree(nnkStmtListExpr, newTree(nnkBlockStmt, newEmptyNode(), blockStmts)) + proc createRoutine(name, query: NimNode; k: NimNodeKind): NimNode = expectKind query, nnkStmtList expectMinLen query, 1 diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim new file mode 100644 index 0000000..c4a2389 --- /dev/null +++ b/tests/ttransactions.nim @@ -0,0 +1,78 @@ +import unittest, os, strformat +import ormin +import ormin/db_utils +when NimVersion < "1.2.0": import ./compat + +let testDir = currentSourcePath.parentDir() + +when defined postgre: + when defined(macosx): + {.passL: "-Wl,-rpath,/opt/homebrew/lib/postgresql@14".} + from db_connector/db_postgres import exec, getValue + const backend = DbBackend.postgre + importModel(backend, "forum_model_postgres") + const sqlFileName = "forum_model_postgres.sql" + let db {.global.} = open("localhost", "test", "test", "test_ormin") +else: + from db_connector/db_sqlite import exec, getValue + const backend = DbBackend.sqlite + importModel(backend, "forum_model_sqlite") + const sqlFileName = "forum_model_sqlite.sql" + var memoryPath = testDir & "/" & ":memory:" + let db {.global.} = open(memoryPath, "", "", "") + +let sqlFilePath = testDir & "/" & sqlFileName + +suite &"Transactions ({backend})": + # Fresh schema + db.dropTable(sqlFilePath) + db.createTable(sqlFilePath) + + test "commit on success": + transaction: + query: + insert person(id = ?(101), name = ?"john101", password = ?"p101", email = ?"john101@mail.com", salt = ?"s101", status = ?"ok") + check db.getValue(sql"select count(*) from person where id = 101") == "1" + + test "rollback on error": + # prepare one row + query: + insert person(id = ?(201), name = ?"john201", password = ?"p201", email = ?"john201@mail.com", salt = ?"s201", status = ?"ok") + # in transaction insert a new row and then violate PK + try: + discard transaction: + query: + insert person(id = ?(202), name = ?"john202", password = ?"p202", email = ?"john202@mail.com", salt = ?"s202", status = ?"ok") + # duplicate key error + query: + insert person(id = ?(201), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") + check false # should not reach + except: + discard + # both inserts inside the transaction should be rolled back + check db.getValue(sql"select count(*) from person where id = 202") == "0" + check db.getValue(sql"select count(*) from person where id = 201 and name = 'dup'") == "0" + + test "tryTransaction returns false on DbError": + let ok = tryTransaction: + query: + insert person(id = ?(301), name = ?"john301", password = ?"p301", email = ?"john301@mail.com", salt = ?"s301", status = ?"ok") + query: + insert person(id = ?(301), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") + check ok == false + check db.getValue(sql"select count(*) from person where id = 301") == "0" + + test "nested savepoints": + transaction: + query: + insert person(id = ?(401), name = ?"john401", password = ?"p401", email = ?"john401@mail.com", salt = ?"s401", status = ?"ok") + let innerOk = tryTransaction: + query: + insert person(id = ?(402), name = ?"john402", password = ?"p402", email = ?"john402@mail.com", salt = ?"s402", status = ?"ok") + query: + insert person(id = ?(401), name = ?"dup401", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") + check innerOk == false + # after inner rollback, we can still insert another row and commit outer + query: + insert person(id = ?(403), name = ?"john403", password = ?"p403", email = ?"john403@mail.com", salt = ?"s403", status = ?"ok") + check db.getValue(sql"select count(*) from person where id in (401,402,403)") == "2" From 3856f35d7892d5e24a196135b8eb3cf7c43e2356 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 02:32:39 -0600 Subject: [PATCH 24/41] fix transaction tests --- ormin/queries.nim | 19 +++++++------------ tests/ttransactions.nim | 4 ++-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 413d8a5..4ad29bb 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1116,9 +1116,7 @@ macro transaction*(body: untyped): untyped = ## Supports nesting via savepoints. let topSym = genSym(nskVar, "orminTop") let spSym = genSym(nskLet, "orminSp") - let rvSym = genSym(nskLet, "orminRv") result = newStmtList() - # block to ensure proper expression value propagation var blockStmts = newStmtList() blockStmts.add newVarStmt(topSym, newTree(nnkInfix, ident"==", ident"txDepth", newLit 0)) blockStmts.add newCall(ident"inc", ident"txDepth") @@ -1142,9 +1140,6 @@ macro transaction*(body: untyped): untyped = ) ) - # body result - blockStmts.add newLetStmt(rvSym, newTree(nnkBlockStmt, newEmptyNode(), body)) - # COMMIT or RELEASE SAVEPOINT var commitStmt = newStmtList() commitStmt.add newTree(nnkWhenStmt, @@ -1163,11 +1158,7 @@ macro transaction*(body: untyped): untyped = ) ) - # try/except wrapping - let tryBody = newStmtList(beginStmt, newEmptyNode()) - # after begin, execute; above rv already holds value - tryBody.add commitStmt - + # Rollback blocks (DbError and any Exception) and rethrow let exceptDb = newTree(nnkExceptBranch, newTree(nnkRefTy, ident"DbError"), newStmtList( newTree(nnkWhenStmt, @@ -1209,10 +1200,14 @@ macro transaction*(body: untyped): untyped = ) ) let finallyStmtTx = newTree(nnkFinally, newStmtList(newCall(ident"dec", ident"txDepth"))) + + # Execute BEGIN; body; COMMIT inside try + let bodyBlk = newTree(nnkBlockStmt, newEmptyNode(), body) + let tryBody = newStmtList(beginStmt, bodyBlk, commitStmt) let tryStmt = newTree(nnkTryStmt, tryBody, exceptDb, exceptAny, finallyStmtTx) blockStmts.add tryStmt - blockStmts.add rvSym - result = newTree(nnkStmtListExpr, newTree(nnkBlockStmt, newEmptyNode(), blockStmts)) + # This macro is statement-level; no resulting expression + result = newTree(nnkStmtList, newTree(nnkBlockStmt, newEmptyNode(), blockStmts)) macro tryTransaction*(body: untyped): untyped = ## Same as `transaction` but returns bool and swallows DbError. diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index c4a2389..980a48a 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -21,7 +21,7 @@ else: var memoryPath = testDir & "/" & ":memory:" let db {.global.} = open(memoryPath, "", "", "") -let sqlFilePath = testDir & "/" & sqlFileName +var sqlFilePath = Path(testDir & "/" & sqlFileName) suite &"Transactions ({backend})": # Fresh schema @@ -40,7 +40,7 @@ suite &"Transactions ({backend})": insert person(id = ?(201), name = ?"john201", password = ?"p201", email = ?"john201@mail.com", salt = ?"s201", status = ?"ok") # in transaction insert a new row and then violate PK try: - discard transaction: + transaction: query: insert person(id = ?(202), name = ?"john202", password = ?"p202", email = ?"john202@mail.com", salt = ?"s202", status = ?"ok") # duplicate key error From df938c5154f238211a6f6603116c602303374edc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 02:38:33 -0600 Subject: [PATCH 25/41] switch to templates --- ormin/queries.nim | 301 +++++++++++++++++----------------------------- 1 file changed, 111 insertions(+), 190 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 4ad29bb..8133985 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1110,203 +1110,124 @@ macro tryQuery*(body: untyped): untyped = # Transactions DSL # ------------------------- -macro transaction*(body: untyped): untyped = +template transaction*(body: untyped) = ## Runs the body inside a database transaction. Commits on success, - ## rolls back on any exception and rethrows. - ## Supports nesting via savepoints. - let topSym = genSym(nskVar, "orminTop") - let spSym = genSym(nskLet, "orminSp") - result = newStmtList() - var blockStmts = newStmtList() - blockStmts.add newVarStmt(topSym, newTree(nnkInfix, ident"==", ident"txDepth", newLit 0)) - blockStmts.add newCall(ident"inc", ident"txDepth") - blockStmts.add newLetStmt(spSym, newTree(nnkInfix, ident"&", newLit("ormin_tx_"), newCall(ident"$", ident"txDepth"))) - - # BEGIN or SAVEPOINT - var beginStmt = newStmtList() - beginStmt.add newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("begin")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("begin")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) - ) - ) - ) - - # COMMIT or RELEASE SAVEPOINT - var commitStmt = newStmtList() - commitStmt.add newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("commit")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("commit")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) - ) - ) - ) + ## rolls back on any exception and rethrows. Supports nesting via savepoints. + block: + let orminTop = txDepth == 0 + inc txDepth + let orminSp = "ormin_tx_" & $txDepth + try: + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("begin") + else: + execNoRowsLoose("savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("begin") + else: + execNoRowsStrict("savepoint " & orminSp) - # Rollback blocks (DbError and any Exception) and rethrow - let exceptDb = newTree(nnkExceptBranch, newTree(nnkRefTy, ident"DbError"), - newStmtList( - newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ) - ), - newTree(nnkRaiseStmt, newEmptyNode()) - ) - ) - let exceptAny = newTree(nnkExceptBranch, ident"Exception", - newStmtList( - newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ) - ), - newTree(nnkRaiseStmt, newEmptyNode()) - ) - ) - let finallyStmtTx = newTree(nnkFinally, newStmtList(newCall(ident"dec", ident"txDepth"))) + block: + body - # Execute BEGIN; body; COMMIT inside try - let bodyBlk = newTree(nnkBlockStmt, newEmptyNode(), body) - let tryBody = newStmtList(beginStmt, bodyBlk, commitStmt) - let tryStmt = newTree(nnkTryStmt, tryBody, exceptDb, exceptAny, finallyStmtTx) - blockStmts.add tryStmt - # This macro is statement-level; no resulting expression - result = newTree(nnkStmtList, newTree(nnkBlockStmt, newEmptyNode(), blockStmts)) + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("commit") + else: + execNoRowsLoose("release savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("commit") + else: + execNoRowsStrict("release savepoint " & orminSp) + except DbError: + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("rollback") + else: + execNoRowsLoose("rollback to savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("rollback") + else: + execNoRowsStrict("rollback to savepoint " & orminSp) + raise + except Exception: + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("rollback") + else: + execNoRowsLoose("rollback to savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("rollback") + else: + execNoRowsStrict("rollback to savepoint " & orminSp) + raise + finally: + dec txDepth -macro tryTransaction*(body: untyped): untyped = +template tryTransaction*(body: untyped): untyped = ## Same as `transaction` but returns bool and swallows DbError. - let topSym = genSym(nskVar, "orminTop") - let spSym = genSym(nskLet, "orminSp") - let okSym = genSym(nskVar, "orminOk") - result = newStmtList() - var blockStmts = newStmtList() - blockStmts.add newVarStmt(topSym, newTree(nnkInfix, ident"==", ident"txDepth", newLit 0)) - blockStmts.add newCall(ident"inc", ident"txDepth") - blockStmts.add newLetStmt(spSym, newTree(nnkInfix, ident"&", newLit("ormin_tx_"), newCall(ident"$", ident"txDepth"))) - blockStmts.add newTree(nnkVarSection, newIdentDefs(okSym, ident"bool", newLit(true))) - - # BEGIN or SAVEPOINT - var beginStmt = newStmtList() - beginStmt.add newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("begin")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("begin")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("savepoint "), spSym)))) - ) - ) - ) + block: + let orminTop = txDepth == 0 + inc txDepth + let orminSp = "ormin_tx_" & $txDepth + var orminOk = true + try: + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("begin") + else: + execNoRowsLoose("savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("begin") + else: + execNoRowsStrict("savepoint " & orminSp) - # Body execution as statements; ignore any result - let bodyBlk = newTree(nnkBlockStmt, newEmptyNode(), body) - - # COMMIT or RELEASE SAVEPOINT - var commitStmt = newStmtList() - commitStmt.add newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("commit")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("commit")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("release savepoint "), spSym)))) - ) - ) - ) + block: + body - let tryBody = newStmtList(beginStmt, bodyBlk, commitStmt) - - let exceptDb = newTree(nnkExceptBranch, ident"DbError", - newStmtList( - newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ) - ), - newAssignment(okSym, newLit(false)) - ) - ) - let exceptAny = newTree(nnkExceptBranch, ident"Exception", - newStmtList( - newTree(nnkWhenStmt, - newTree(nnkElifBranch, - newTree(nnkInfix, ident"==", ident"dbBackend", newTree(nnkDotExpr, ident"DbBackend", ident"postgre")), - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsLoose", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsLoose", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ), - newTree(nnkElse, - newTree(nnkIfStmt, - newTree(nnkElifBranch, topSym, newStmtList(newCall(ident"execNoRowsStrict", newLit("rollback")))), - newTree(nnkElse, newStmtList(newCall(ident"execNoRowsStrict", newTree(nnkInfix, ident"&", newLit("rollback to savepoint "), spSym)))) - ) - ) - ), - newTree(nnkRaiseStmt, newEmptyNode()) - ) - ) - let finallyStmt = newTree(nnkFinally, newStmtList(newCall(ident"dec", ident"txDepth"))) - let tryStmt = newTree(nnkTryStmt, tryBody, exceptDb, exceptAny, finallyStmt) - blockStmts.add tryStmt - blockStmts.add okSym - result = newTree(nnkStmtListExpr, newTree(nnkBlockStmt, newEmptyNode(), blockStmts)) + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("commit") + else: + execNoRowsLoose("release savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("commit") + else: + execNoRowsStrict("release savepoint " & orminSp) + except DbError: + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("rollback") + else: + execNoRowsLoose("rollback to savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("rollback") + else: + execNoRowsStrict("rollback to savepoint " & orminSp) + orminOk = false + except Exception: + when dbBackend == DbBackend.postgre: + if orminTop: + execNoRowsLoose("rollback") + else: + execNoRowsLoose("rollback to savepoint " & orminSp) + else: + if orminTop: + execNoRowsStrict("rollback") + else: + execNoRowsStrict("rollback to savepoint " & orminSp) + raise + finally: + dec txDepth + orminOk proc createRoutine(name, query: NimNode; k: NimNodeKind): NimNode = expectKind query, nnkStmtList From 398df318f69bde6e9247c54f8c95ef55fc47eee9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 02:53:53 -0600 Subject: [PATCH 26/41] switch to templates --- ormin/queries.nim | 85 +++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 47 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 8133985..25b6720 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -141,7 +141,19 @@ type retExpr: NimNode # Transaction state for nested transactions -var txDepth* {.threadvar.}: int +var txDepth {.threadvar.}: int + +proc getTxDepth*(): int = + return txDepth + +proc isTopTx*(): int = + return txDepth == 0 + +proc incTxDepth*() = + inc txDepth + +proc decTxDepth*() = + inc txDepth # Execute a non-row SQL statement strictly (errors on failure) template execNoRowsStrict*(sqlStmt: string) = @@ -1114,60 +1126,39 @@ template transaction*(body: untyped) = ## Runs the body inside a database transaction. Commits on success, ## rolls back on any exception and rethrows. Supports nesting via savepoints. block: - let orminTop = txDepth == 0 - inc txDepth - let orminSp = "ormin_tx_" & $txDepth + let orminTop = isTopTx() + incTxDepth() + when dbBackend == DbBackend.postgre: + let orminSp = "SAVEPOINT ormin_tx_" & $txDepth + else: + let orminSp = "ormin_tx_" & $txDepth + try: - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("begin") - else: - execNoRowsLoose("savepoint " & orminSp) + if orminTop: + execNoRowsLoose("begin") else: - if orminTop: - execNoRowsStrict("begin") - else: - execNoRowsStrict("savepoint " & orminSp) + execNoRowsLoose("savepoint " & orminSp) - block: - body + `body` - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("commit") - else: - execNoRowsLoose("release savepoint " & orminSp) + if orminTop: + execNoRowsLoose("commit") else: - if orminTop: - execNoRowsStrict("commit") - else: - execNoRowsStrict("release savepoint " & orminSp) - except DbError: - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("rollback") - else: - execNoRowsLoose("rollback to savepoint " & orminSp) + execNoRowsLoose("release savepoint " & orminSp) + except DbError as e: + if orminTop: + execNoRowsLoose("rollback") else: - if orminTop: - execNoRowsStrict("rollback") - else: - execNoRowsStrict("rollback to savepoint " & orminSp) - raise - except Exception: - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("rollback") - else: - execNoRowsLoose("rollback to savepoint " & orminSp) + execNoRowsLoose("rollback to savepoint " & orminSp) + raise e + except Exception as e: + if orminTop: + execNoRowsLoose("rollback") else: - if orminTop: - execNoRowsStrict("rollback") - else: - execNoRowsStrict("rollback to savepoint " & orminSp) - raise + execNoRowsLoose("rollback to savepoint " & orminSp) + raise e finally: - dec txDepth + decTxDepth() template tryTransaction*(body: untyped): untyped = ## Same as `transaction` but returns bool and swallows DbError. From fcbed0784f0d191e6dc36c2d484d56d33453faee Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 02:54:05 -0600 Subject: [PATCH 27/41] switch to templates --- ormin/queries.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 25b6720..97f1829 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -146,7 +146,7 @@ var txDepth {.threadvar.}: int proc getTxDepth*(): int = return txDepth -proc isTopTx*(): int = +proc isTopTx*(): bool = return txDepth == 0 proc incTxDepth*() = From 32cbbba904a1286bd9325106463bbd6b63f9bbe2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:03:45 -0600 Subject: [PATCH 28/41] switch to templates --- ormin/queries.nim | 105 ++++++++++++++++------------------------------ 1 file changed, 36 insertions(+), 69 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 97f1829..831e8aa 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1122,99 +1122,66 @@ macro tryQuery*(body: untyped): untyped = # Transactions DSL # ------------------------- +template txBegin*(sp: untyped) = + if isTopTx(): + execNoRowsLoose("begin") + else: + execNoRowsLoose("savepoint " & sp) + +template txCommit*(sp: untyped) = + if isTopTx(): + execNoRowsLoose("commit") + else: + execNoRowsLoose("release savepoint " & sp) + +template txRollback*(sp: untyped) = + if isTopTx(): + execNoRowsLoose("rollback") + else: + execNoRowsLoose("rollback to savepoint " & sp) + template transaction*(body: untyped) = ## Runs the body inside a database transaction. Commits on success, ## rolls back on any exception and rethrows. Supports nesting via savepoints. block: - let orminTop = isTopTx() incTxDepth() when dbBackend == DbBackend.postgre: - let orminSp = "SAVEPOINT ormin_tx_" & $txDepth + let sp = "SAVEPOINT ormin_tx_" & $txDepth else: - let orminSp = "ormin_tx_" & $txDepth + let sp = "ormin_tx_" & $txDepth try: - if orminTop: - execNoRowsLoose("begin") - else: - execNoRowsLoose("savepoint " & orminSp) - + txBegin(sp) `body` - - if orminTop: - execNoRowsLoose("commit") - else: - execNoRowsLoose("release savepoint " & orminSp) + txCommit(sp) except DbError as e: - if orminTop: - execNoRowsLoose("rollback") - else: - execNoRowsLoose("rollback to savepoint " & orminSp) + txRollback(sp) raise e except Exception as e: - if orminTop: - execNoRowsLoose("rollback") - else: - execNoRowsLoose("rollback to savepoint " & orminSp) + txRollback(sp) raise e finally: decTxDepth() -template tryTransaction*(body: untyped): untyped = +template tryTransaction*(body: untyped): bool = ## Same as `transaction` but returns bool and swallows DbError. block: - let orminTop = txDepth == 0 - inc txDepth - let orminSp = "ormin_tx_" & $txDepth + incTxDepth() + when dbBackend == DbBackend.postgre: + let sp = "SAVEPOINT ormin_tx_" & $txDepth + else: + let sp = "ormin_tx_" & $txDepth + var orminOk = true try: - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("begin") - else: - execNoRowsLoose("savepoint " & orminSp) - else: - if orminTop: - execNoRowsStrict("begin") - else: - execNoRowsStrict("savepoint " & orminSp) - - block: - body - - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("commit") - else: - execNoRowsLoose("release savepoint " & orminSp) - else: - if orminTop: - execNoRowsStrict("commit") - else: - execNoRowsStrict("release savepoint " & orminSp) + txBegin(sp) + `body` + txCommit(sp) except DbError: - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("rollback") - else: - execNoRowsLoose("rollback to savepoint " & orminSp) - else: - if orminTop: - execNoRowsStrict("rollback") - else: - execNoRowsStrict("rollback to savepoint " & orminSp) + txRollback(sp) orminOk = false except Exception: - when dbBackend == DbBackend.postgre: - if orminTop: - execNoRowsLoose("rollback") - else: - execNoRowsLoose("rollback to savepoint " & orminSp) - else: - if orminTop: - execNoRowsStrict("rollback") - else: - execNoRowsStrict("rollback to savepoint " & orminSp) + txRollback(sp) raise finally: dec txDepth From 03bba5a11d8b18c409ab33dde2b6fa0af4f4f772 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:07:43 -0600 Subject: [PATCH 29/41] switch to templates --- ormin/queries.nim | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 831e8aa..760faec 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1124,7 +1124,7 @@ macro tryQuery*(body: untyped): untyped = template txBegin*(sp: untyped) = if isTopTx(): - execNoRowsLoose("begin") + execNoRowsLoose("begin transaction") else: execNoRowsLoose("savepoint " & sp) @@ -1145,10 +1145,7 @@ template transaction*(body: untyped) = ## rolls back on any exception and rethrows. Supports nesting via savepoints. block: incTxDepth() - when dbBackend == DbBackend.postgre: - let sp = "SAVEPOINT ormin_tx_" & $txDepth - else: - let sp = "ormin_tx_" & $txDepth + let sp = "ormin_tx_" & $txDepth try: txBegin(sp) @@ -1167,10 +1164,7 @@ template tryTransaction*(body: untyped): bool = ## Same as `transaction` but returns bool and swallows DbError. block: incTxDepth() - when dbBackend == DbBackend.postgre: - let sp = "SAVEPOINT ormin_tx_" & $txDepth - else: - let sp = "ormin_tx_" & $txDepth + let sp = "ormin_tx_" & $txDepth var orminOk = true try: From e88826958915f5434f2235982c0c7ec8b5496d76 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:21:37 -0600 Subject: [PATCH 30/41] switch to templates --- ormin/queries.nim | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 760faec..64de349 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -140,23 +140,10 @@ type # For SQLite: expression to return instead of last_insert_rowid() retExpr: NimNode -# Transaction state for nested transactions -var txDepth {.threadvar.}: int - -proc getTxDepth*(): int = - return txDepth - -proc isTopTx*(): bool = - return txDepth == 0 - -proc incTxDepth*() = - inc txDepth - -proc decTxDepth*() = - inc txDepth - # Execute a non-row SQL statement strictly (errors on failure) template execNoRowsStrict*(sqlStmt: string) = + when defined(debugOrminSql): + echo "Ormin Executing: ", sqlStmt let s {.gensym.} = prepareStmt(db, sqlStmt) startQuery(db, s) if stepQuery(db, s, false): @@ -167,6 +154,8 @@ template execNoRowsStrict*(sqlStmt: string) = # Execute a non-row SQL statement, relying on startQuery to raise on failure template execNoRowsLoose*(sqlStmt: string) = + when defined(debugOrminSql): + echo "Ormin Executing: ", sqlStmt let s {.gensym.} = prepareStmt(db, sqlStmt) startQuery(db, s) discard stepQuery(db, s, false) @@ -1122,6 +1111,22 @@ macro tryQuery*(body: untyped): untyped = # Transactions DSL # ------------------------- +# Transaction state for nested transactions +var txDepth {.threadvar.}: int + +proc getTxDepth*(): int = + return txDepth + +proc isTopTx*(): bool = + result = txDepth == 1 + echo "IS TOP TX: ", result, " ", txDepth + +proc incTxDepth*() = + inc txDepth + +proc decTxDepth*() = + dec txDepth + template txBegin*(sp: untyped) = if isTopTx(): execNoRowsLoose("begin transaction") @@ -1178,7 +1183,7 @@ template tryTransaction*(body: untyped): bool = txRollback(sp) raise finally: - dec txDepth + decTxDepth() orminOk proc createRoutine(name, query: NimNode; k: NimNodeKind): NimNode = From 49b9815b06006ca816a885a5da3cbef25902a75c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:21:48 -0600 Subject: [PATCH 31/41] switch to templates --- ormin/queries.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 64de349..3b8b5a9 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1115,11 +1115,10 @@ macro tryQuery*(body: untyped): untyped = var txDepth {.threadvar.}: int proc getTxDepth*(): int = - return txDepth + result = txDepth proc isTopTx*(): bool = result = txDepth == 1 - echo "IS TOP TX: ", result, " ", txDepth proc incTxDepth*() = inc txDepth From 000393cb3c8f18cf922011aedd0dad097edbf11c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:26:27 -0600 Subject: [PATCH 32/41] switch to templates --- ormin/queries.nim | 6 +++--- tests/ttransactions.nim | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 3b8b5a9..423f967 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1158,9 +1158,9 @@ template transaction*(body: untyped) = except DbError as e: txRollback(sp) raise e - except Exception as e: + except CatchableError, Defect: txRollback(sp) - raise e + raise finally: decTxDepth() @@ -1178,7 +1178,7 @@ template tryTransaction*(body: untyped): bool = except DbError: txRollback(sp) orminOk = false - except Exception: + except CatchableError, Defect: txRollback(sp) raise finally: diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index 980a48a..019f650 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -47,7 +47,8 @@ suite &"Transactions ({backend})": query: insert person(id = ?(201), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") check false # should not reach - except: + except DbError as e: + echo "GOT: ", $e.name, ": ", $e.msg discard # both inserts inside the transaction should be rolled back check db.getValue(sql"select count(*) from person where id = 202") == "0" From 2177fc47f7bd5748992e9d10b8e3486208a56369 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:42:01 -0600 Subject: [PATCH 33/41] switch to templates --- ormin/ormin_postgre.nim | 2 ++ ormin/ormin_sqlite.nim | 2 ++ ormin/queries.nim | 8 ++++---- tests/ttransactions.nim | 1 - 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ormin/ormin_postgre.nim b/ormin/ormin_postgre.nim index fce1ab9..0d09799 100644 --- a/ormin/ormin_postgre.nim +++ b/ormin/ormin_postgre.nim @@ -33,6 +33,8 @@ proc c_strtol(buf: cstring, endptr: ptr cstring = nil, base: cint = 10): int {. var sid {.compileTime.}: int proc prepareStmt*(db: DbConn; q: string): PStmt = + when defined(debugOrminTrace): + echo "[[Ormin Executing]]: ", q inc sid result = "ormin" & $sid var res = pqprepare(db, result, q, 0, nil) diff --git a/ormin/ormin_sqlite.nim b/ormin/ormin_sqlite.nim index 2f19c89..321a8e0 100644 --- a/ormin/ormin_sqlite.nim +++ b/ormin/ormin_sqlite.nim @@ -28,6 +28,8 @@ proc dbError*(db: DbConn) {.noreturn.} = raise e proc prepareStmt*(db: DbConn; q: string): PStmt = + when defined(debugOrminTrace): + echo "[[Ormin Executing]]: ", q if prepare_v2(db, q, q.len.cint, result, nil) != SQLITE_OK: dbError(db) diff --git a/ormin/queries.nim b/ormin/queries.nim index 423f967..69da001 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -142,8 +142,8 @@ type # Execute a non-row SQL statement strictly (errors on failure) template execNoRowsStrict*(sqlStmt: string) = - when defined(debugOrminSql): - echo "Ormin Executing: ", sqlStmt + when defined(debugOrminTrace): + echo "[[Ormin Executing]]: ", q let s {.gensym.} = prepareStmt(db, sqlStmt) startQuery(db, s) if stepQuery(db, s, false): @@ -154,8 +154,8 @@ template execNoRowsStrict*(sqlStmt: string) = # Execute a non-row SQL statement, relying on startQuery to raise on failure template execNoRowsLoose*(sqlStmt: string) = - when defined(debugOrminSql): - echo "Ormin Executing: ", sqlStmt + when defined(debugOrminTrace): + echo "[[Ormin Executing]]: ", sqlStmt let s {.gensym.} = prepareStmt(db, sqlStmt) startQuery(db, s) discard stepQuery(db, s, false) diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index 019f650..00c0909 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -48,7 +48,6 @@ suite &"Transactions ({backend})": insert person(id = ?(201), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") check false # should not reach except DbError as e: - echo "GOT: ", $e.name, ": ", $e.msg discard # both inserts inside the transaction should be rolled back check db.getValue(sql"select count(*) from person where id = 202") == "0" From bde349d9c901771b755a53c56d9b353ffa865ec6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:43:06 -0600 Subject: [PATCH 34/41] switch to templates --- tests/ttransactions.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index 00c0909..9482bae 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -23,10 +23,11 @@ else: var sqlFilePath = Path(testDir & "/" & sqlFileName) +# Fresh schema +db.dropTable(sqlFilePath) +db.createTable(sqlFilePath) + suite &"Transactions ({backend})": - # Fresh schema - db.dropTable(sqlFilePath) - db.createTable(sqlFilePath) test "commit on success": transaction: From 862da5aff3a563510a06bacb0461ac20d1a4f458 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:45:54 -0600 Subject: [PATCH 35/41] switch to templates --- ormin/queries.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 69da001..35721a2 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1155,9 +1155,9 @@ template transaction*(body: untyped) = txBegin(sp) `body` txCommit(sp) - except DbError as e: + except DbError: txRollback(sp) - raise e + raise except CatchableError, Defect: txRollback(sp) raise From 201856fed0e373f3f8baa4cd5c1e596407cb82cb Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 03:58:38 -0600 Subject: [PATCH 36/41] update tests --- ormin/queries.nim | 12 +++++++----- tests/ttransactions.nim | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 35721a2..981e00f 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1164,26 +1164,28 @@ template transaction*(body: untyped) = finally: decTxDepth() -template tryTransaction*(body: untyped): bool = - ## Same as `transaction` but returns bool and swallows DbError. +macro getBlock(blk: untyped): untyped = + result = blk[0] + +template transaction*(body, other: untyped) = + ## Runs the body inside a database transaction. Commits on success, + ## rolls back on any exception and rethrows. Supports nesting via savepoints. block: incTxDepth() let sp = "ormin_tx_" & $txDepth - var orminOk = true try: txBegin(sp) `body` txCommit(sp) except DbError: txRollback(sp) - orminOk = false + getBlock(`other`) except CatchableError, Defect: txRollback(sp) raise finally: decTxDepth() - orminOk proc createRoutine(name, query: NimNode; k: NimNodeKind): NimNode = expectKind query, nnkStmtList diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index 9482bae..187b3c9 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -54,26 +54,58 @@ suite &"Transactions ({backend})": check db.getValue(sql"select count(*) from person where id = 202") == "0" check db.getValue(sql"select count(*) from person where id = 201 and name = 'dup'") == "0" - test "tryTransaction returns false on DbError": - let ok = tryTransaction: + test "rollback on error": + # prepare one row + var failed = false + query: + insert person(id = ?(501), name = ?"john501", password = ?"p501", email = ?"john501@mail.com", salt = ?"s501", status = ?"ok") + # in transaction insert a new row and then violate PK + transaction: + query: + insert person(id = ?(502), name = ?"john502", password = ?"p502", email = ?"john502@mail.com", salt = ?"s502", status = ?"ok") + # duplicate key error + query: + insert person(id = ?(501), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") + check false # should not reach + else: + failed = true + + check failed + # both inserts inside the transaction should be rolled back + check db.getValue(sql"select count(*) from person where id = 502") == "0" + check db.getValue(sql"select count(*) from person where id = 501 and name = 'dup'") == "0" + + test "transaction set false on DbError": + var failed = false + transaction: query: insert person(id = ?(301), name = ?"john301", password = ?"p301", email = ?"john301@mail.com", salt = ?"s301", status = ?"ok") query: insert person(id = ?(301), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") - check ok == false + else: + failed = true + check failed check db.getValue(sql"select count(*) from person where id = 301") == "0" test "nested savepoints": + var failed = false transaction: query: insert person(id = ?(401), name = ?"john401", password = ?"p401", email = ?"john401@mail.com", salt = ?"s401", status = ?"ok") - let innerOk = tryTransaction: + var innerOk = true + transaction: query: insert person(id = ?(402), name = ?"john402", password = ?"p402", email = ?"john402@mail.com", salt = ?"s402", status = ?"ok") query: insert person(id = ?(401), name = ?"dup401", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") + else: + innerOk = false + check innerOk == false + # after inner rollback, we can still insert another row and commit outer query: insert person(id = ?(403), name = ?"john403", password = ?"p403", email = ?"john403@mail.com", salt = ?"s403", status = ?"ok") + else: + failed = true check db.getValue(sql"select count(*) from person where id in (401,402,403)") == "2" From 23d8e290a4259f0a78540d118fa20ce3535649c9 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 04:03:25 -0600 Subject: [PATCH 37/41] update tests --- tests/ttransactions.nim | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index 187b3c9..57a0538 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -54,7 +54,7 @@ suite &"Transactions ({backend})": check db.getValue(sql"select count(*) from person where id = 202") == "0" check db.getValue(sql"select count(*) from person where id = 201 and name = 'dup'") == "0" - test "rollback on error": + test "rollback on error with else": # prepare one row var failed = false query: @@ -75,6 +75,26 @@ suite &"Transactions ({backend})": check db.getValue(sql"select count(*) from person where id = 502") == "0" check db.getValue(sql"select count(*) from person where id = 501 and name = 'dup'") == "0" + test "commit normally with else": + # prepare one row + var failed = false + query: + insert person(id = ?(601), name = ?"john601", password = ?"p601", email = ?"john601@mail.com", salt = ?"s601", status = ?"ok") + # in transaction insert a new row and then violate PK + transaction: + query: + insert person(id = ?(602), name = ?"john602", password = ?"p602", email = ?"john602@mail.com", salt = ?"s602", status = ?"ok") + query: + insert person(id = ?(603), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") + else: + failed = true + + check not failed + # both inserts inside the transaction should be rolled back + check db.getValue(sql"select count(*) from person where id = 603") == "1" + check db.getValue(sql"select count(*) from person where id = 602") == "1" + check db.getValue(sql"select count(*) from person where id = 601") == "1" + test "transaction set false on DbError": var failed = false transaction: From 9a704cf2b0c4987665d8def288daa06abc211985 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 04:04:44 -0600 Subject: [PATCH 38/41] update tests --- tests/ttransactions.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index 57a0538..9d84446 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -68,6 +68,7 @@ suite &"Transactions ({backend})": insert person(id = ?(501), name = ?"dup", password = ?"p", email = ?"e", salt = ?"s", status = ?"x") check false # should not reach else: + echo "do something else..." failed = true check failed From 98b1baa6093d562de86b2ec20bbc970563c9738c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 23:18:58 -0600 Subject: [PATCH 39/41] Update config.nims Co-authored-by: Andreas Rumpf --- config.nims | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.nims b/config.nims index 2c1cd58..08eca33 100644 --- a/config.nims +++ b/config.nims @@ -31,9 +31,9 @@ task test_postgres, "Run PostgreSQL test suite": exec "./tools/ormin_importer tests/forum_model_postgres.sql" exec "./tools/ormin_importer tests/model_postgre.sql" - exec "nim c -f -d:nimDebugDlOpen -f -d:nimDebugDlOpen -r -d:postgre tests/tfeature" - exec "nim c -f -d:nimDebugDlOpen -f -d:nimDebugDlOpen -r -d:postgre tests/tcommon" - exec "nim c -f -d:nimDebugDlOpen -f -d:nimDebugDlOpen -r -d:postgre tests/tpostgre" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tfeature" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tcommon" + exec "nim c -f -d:nimDebugDlOpen -r -d:postgre tests/tpostgre" task buildexamples, "Build examples: chat and forum": buildimporterTask() From cedcb8f9fcb3cad2eb253fba4a514a2e1eeb5f22 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 Oct 2025 23:21:39 -0600 Subject: [PATCH 40/41] Update ormin/queries.nim Co-authored-by: Andreas Rumpf --- ormin/queries.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 981e00f..e4e82ff 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -153,7 +153,7 @@ template execNoRowsStrict*(sqlStmt: string) = dbError(db) # Execute a non-row SQL statement, relying on startQuery to raise on failure -template execNoRowsLoose*(sqlStmt: string) = +template execNoRowsLoose(sqlStmt: string) = when defined(debugOrminTrace): echo "[[Ormin Executing]]: ", sqlStmt let s {.gensym.} = prepareStmt(db, sqlStmt) From 0328899904dd8c7ad026356c7d91d48313787fec Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Oct 2025 09:25:38 -0600 Subject: [PATCH 41/41] update tests --- tests/ttransactions.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ttransactions.nim b/tests/ttransactions.nim index 9d84446..35d0834 100644 --- a/tests/ttransactions.nim +++ b/tests/ttransactions.nim @@ -35,7 +35,7 @@ suite &"Transactions ({backend})": insert person(id = ?(101), name = ?"john101", password = ?"p101", email = ?"john101@mail.com", salt = ?"s101", status = ?"ok") check db.getValue(sql"select count(*) from person where id = 101") == "1" - test "rollback on error": + test "rollback on error with manual try except": # prepare one row query: insert person(id = ?(201), name = ?"john201", password = ?"p201", email = ?"john201@mail.com", salt = ?"s201", status = ?"ok")