From 59bc1778395926759b0453fe99321bbc8b8cc67a Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Tue, 23 Apr 2024 23:58:34 +0200 Subject: [PATCH 1/9] Add sql.js fallback for sqlite in wasm --- sqlite/lib/SQLiteService.js | 13 ++++-- sqlite/lib/sql.js.js | 84 +++++++++++++++++++++++++++++++++++++ sqlite/package.json | 5 ++- 3 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 sqlite/lib/sql.js.js diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index 2320b6e7e..f11354562 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -1,6 +1,12 @@ const { SQLService } = require('@cap-js/db-service') const cds = require('@sap/cds/lib') -const sqlite = require('better-sqlite3') +let sqlite +try { + sqlite = require('better-sqlite3') +} catch (err) { + // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite) + sqlite = require('./sql.js.js') +} const $session = Symbol('dbc.session') const convStrm = require('stream/consumers') const { Readable } = require('stream') @@ -20,9 +26,10 @@ class SQLiteService extends SQLService { get factory() { return { options: { max: 1, ...this.options.pool }, - create: tenant => { + create: async tenant => { const database = this.url4(tenant) const dbc = new sqlite(database) + await dbc.ready const deterministic = { deterministic: true } dbc.function('session_context', key => dbc[$session][key]) @@ -224,7 +231,7 @@ class SQLiteService extends SQLService { // int64 is stored as native int64 for best comparison // Reading int64 as string to not loose precision Int64: expr => `CAST(${expr} as TEXT)`, - + // Reading decimal as string to not loose precision Decimal: expr => `CAST(${expr} as TEXT)`, diff --git a/sqlite/lib/sql.js.js b/sqlite/lib/sql.js.js new file mode 100644 index 000000000..d62626524 --- /dev/null +++ b/sqlite/lib/sql.js.js @@ -0,0 +1,84 @@ +const initSqlJs = require('sql.js'); + +const init = initSqlJs({}) + +class WasmSqlite { + constructor(database) { + // TODO: load / store database file contents + this.ready = init + .then(SQL => { this.db = new SQL.Database() }) + + this.memory = true + this.gc = new FinalizationRegistry(stmt => { stmt.free() }) + } + + prepare(sql) { + const stmt = this.db.prepare(sql) + const ret = { + run: (params) => { + try { + stmt.bind(params) + stmt.step() + return { changes: this.db.getRowsModified(stmt) } + } catch (err) { + if (err.message.indexOf('NOT NULL constraint failed:') === 0) { + err.code = 'SQLITE_CONSTRAINT_NOTNULL' + } + throw err + } + }, + get: (params) => { + const columns = stmt.getColumnNames() + stmt.bind(params) + stmt.step() + const row = stmt.get() + const ret = {} + for (let i = 0; i < columns.length; i++) { + ret[columns[i]] = row[i] + } + return ret + }, + all: (params) => { + const columns = stmt.getColumnNames() + const ret = [] + stmt.bind(params) + while (stmt.step()) { + const row = stmt.get() + const obj = {} + for (let i = 0; i < columns.length; i++) { + obj[columns[i]] = row[i] + } + ret.push(obj) + } + return ret + } + } + this.gc.register(ret, stmt) + return ret + } + + exec(sql) { + try { + const { columns, values } = this.db.exec(sql) + return !Array.isArray(values) ? values : values.map(val => { + const ret = {} + for (let i = 0; i < columns.length; i++) { + ret[columns[i]] = val[i] + } + return ret + }) + } catch (err) { + // REVISIT: address transaction errors + if (sql === 'BEGIN' || sql === 'ROLLBACK') { return } + throw err + } + } + + function(name, config, func) { + this.db.create_function(name, func || config) + } + + close() { this.db.close() } +} + +module.exports = WasmSqlite diff --git a/sqlite/package.json b/sqlite/package.json index cde98fc78..0c8a69797 100644 --- a/sqlite/package.json +++ b/sqlite/package.json @@ -33,6 +33,9 @@ "@cap-js/db-service": "^1.7.0", "better-sqlite3": "^9.3.0" }, + "optionalDependencies": { + "sql.js": "^1.10.3" + }, "peerDependencies": { "@sap/cds": ">=7.6" }, @@ -52,4 +55,4 @@ } }, "license": "SEE LICENSE" -} +} \ No newline at end of file From f456747ceb77e3485aac851596046a7af0713096 Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 08:56:17 +0200 Subject: [PATCH 2/9] Update package.json --- package-lock.json | 9 +++++++++ sqlite/package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 87d00ebad..fbe838b9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6064,6 +6064,12 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/sql.js": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.10.3.tgz", + "integrity": "sha512-H46aWtQkdyjZwFQgraUruy5h/DyJBbAK3EA/WEMqiqF6PGPfKBSKBj/er3dVyYqVIoYfRf5TFM/loEjtQIrqJg==", + "optional": true + }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -6619,6 +6625,9 @@ "node": ">=16", "npm": ">=8" }, + "optionalDependencies": { + "sql.js": "^1.10.3" + }, "peerDependencies": { "@sap/cds": ">=7.6" } diff --git a/sqlite/package.json b/sqlite/package.json index 0c8a69797..6f411b396 100644 --- a/sqlite/package.json +++ b/sqlite/package.json @@ -55,4 +55,4 @@ } }, "license": "SEE LICENSE" -} \ No newline at end of file +} From 3a56fe172cb78b00f2a6ddcf9cfa5371b562bf04 Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:03:02 +0200 Subject: [PATCH 3/9] Adding sql.js test pipeline --- .github/workflows/sqlite-wasm.yml | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/sqlite-wasm.yml diff --git a/.github/workflows/sqlite-wasm.yml b/.github/workflows/sqlite-wasm.yml new file mode 100644 index 000000000..62ba3b0f0 --- /dev/null +++ b/.github/workflows/sqlite-wasm.yml @@ -0,0 +1,37 @@ +name: Tests WASM + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened, auto_merge_enabled] + +# Allow parallel jobs on `main`, so that each commit is tested. For PRs, run only the latest commit. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 5 + name: Node.js ${{ matrix.node }} + + strategy: + fail-fast: true + matrix: + node: [18] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: 'npm' + + - run: npm ci + # Remove better-sqlite3 to force switching to sql.js + - run: npm install sql.js && rm -rf /node_modules/better-sqlite3/ + - run: npm test -w sqlite -- --maxWorkers=1 + env: + FORCE_COLOR: true From e44a447a039a148f04d26515e66d4d238630526a Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:08:14 +0200 Subject: [PATCH 4/9] Double checking that the fallback is tested --- sqlite/lib/SQLiteService.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index f11354562..79c9507f8 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -3,7 +3,9 @@ const cds = require('@sap/cds/lib') let sqlite try { sqlite = require('better-sqlite3') + process.stdout.write('Using default sqlite driver better-sqlite3\n') } catch (err) { + process.stdout.write('Using fallback sqlite driver sql.js\n') // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite) sqlite = require('./sql.js.js') } From 9b1281a12cf05ece6c965220227a3b52d864a7f2 Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:09:30 +0200 Subject: [PATCH 5/9] Adjust better-sqlite3 removal --- .github/workflows/sqlite-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sqlite-wasm.yml b/.github/workflows/sqlite-wasm.yml index 62ba3b0f0..1fca0ff55 100644 --- a/.github/workflows/sqlite-wasm.yml +++ b/.github/workflows/sqlite-wasm.yml @@ -31,7 +31,7 @@ jobs: - run: npm ci # Remove better-sqlite3 to force switching to sql.js - - run: npm install sql.js && rm -rf /node_modules/better-sqlite3/ + - run: npm install sql.js && rm -rf node_modules/better-sqlite3/ - run: npm test -w sqlite -- --maxWorkers=1 env: FORCE_COLOR: true From 3ebf2f4657f05901f22037999bd72728156c3bef Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 24 Apr 2024 09:11:08 +0200 Subject: [PATCH 6/9] Remove debug logging --- sqlite/lib/SQLiteService.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index 79c9507f8..f11354562 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -3,9 +3,7 @@ const cds = require('@sap/cds/lib') let sqlite try { sqlite = require('better-sqlite3') - process.stdout.write('Using default sqlite driver better-sqlite3\n') } catch (err) { - process.stdout.write('Using fallback sqlite driver sql.js\n') // When failing to load better-sqlite3 it fallsback to sql.js (wasm version of sqlite) sqlite = require('./sql.js.js') } From 6546a7d373987e5ac6473b6b4ce202f2bdc9816e Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Tue, 28 May 2024 10:48:15 +0200 Subject: [PATCH 7/9] Remove linting warning --- sqlite/lib/sql.js.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite/lib/sql.js.js b/sqlite/lib/sql.js.js index d62626524..3fb1119ac 100644 --- a/sqlite/lib/sql.js.js +++ b/sqlite/lib/sql.js.js @@ -3,7 +3,7 @@ const initSqlJs = require('sql.js'); const init = initSqlJs({}) class WasmSqlite { - constructor(database) { + constructor(/*database*/) { // TODO: load / store database file contents this.ready = init .then(SQL => { this.db = new SQL.Database() }) From 19f4043859f978bbde4fb522920b14816e238729 Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 17 Sep 2025 11:07:01 +0200 Subject: [PATCH 8/9] Bump sql.js version and fix streaming support --- package-lock.json | 42 +++++++------------------------------ sqlite/lib/SQLiteService.js | 6 ++---- sqlite/lib/sql.js.js | 14 +++++++++---- sqlite/package.json | 11 ++++++---- 4 files changed, 27 insertions(+), 46 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4b888e264..24b71a4b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ }, "db-service": { "name": "@cap-js/db-service", - "version": "2.3.0", + "version": "2.4.0", "license": "Apache-2.0", "dependencies": { "generic-pool": "^3.9.0" @@ -32,7 +32,7 @@ }, "hana": { "name": "@cap-js/hana", - "version": "2.1.2", + "version": "2.2.0", "license": "Apache-2.0", "dependencies": { "@cap-js/db-service": "^2.1.1", @@ -1739,39 +1739,13 @@ "node": ">= 10.x" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true - }, "node_modules/sql.js": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.10.3.tgz", - "integrity": "sha512-H46aWtQkdyjZwFQgraUruy5h/DyJBbAK3EA/WEMqiqF6PGPfKBSKBj/er3dVyYqVIoYfRf5TFM/loEjtQIrqJg==", + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.13.0.tgz", + "integrity": "sha512-RJbVP1HRDlUUXahJ7VMTcu9Rm1Nzw+EBpoPr94vnbD4LwR715F3CcxE2G2k45PewcaZ57pjetYa+LoSJLAASgA==", + "license": "MIT", "optional": true }, - "node_modules/stack-utils": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", - "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", - "dev": true, - "dependencies": { - "escape-string-regexp": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/stack-utils/node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -1927,7 +1901,7 @@ }, "postgres": { "name": "@cap-js/postgres", - "version": "2.0.4", + "version": "2.0.5", "license": "Apache-2.0", "dependencies": { "@cap-js/db-service": "^2", @@ -1952,7 +1926,7 @@ "better-sqlite3": "^12.0.0" }, "optionalDependencies": { - "sql.js": "^1.10.3" + "sql.js": "^1.13.0" }, "peerDependencies": { "@sap/cds": ">=9" diff --git a/sqlite/lib/SQLiteService.js b/sqlite/lib/SQLiteService.js index 473ff208f..9b2f84105 100644 --- a/sqlite/lib/SQLiteService.js +++ b/sqlite/lib/SQLiteService.js @@ -143,10 +143,8 @@ class SQLiteService extends SQLService { } async _allStream(stmt, binding_params, one, objectMode) { - stmt = stmt.constructor.name === 'Statement' ? stmt : stmt.__proto__ - stmt.raw(true) - const get = stmt.get(binding_params) - if (!get) return [] + stmt = stmt.iterate ? stmt : stmt.__proto__ + stmt.raw?.(true) const rs = stmt.iterate(binding_params) const stream = Readable.from(objectMode ? this._iteratorObjectMode(rs) : this._iteratorRaw(rs, one), { objectMode }) const close = () => rs.return() // finish result set when closed early diff --git a/sqlite/lib/sql.js.js b/sqlite/lib/sql.js.js index 3fb1119ac..8c9be320d 100644 --- a/sqlite/lib/sql.js.js +++ b/sqlite/lib/sql.js.js @@ -15,11 +15,11 @@ class WasmSqlite { prepare(sql) { const stmt = this.db.prepare(sql) const ret = { - run: (params) => { + run(params) { try { stmt.bind(params) stmt.step() - return { changes: this.db.getRowsModified(stmt) } + return { changes: stmt.db.getRowsModified(stmt) } } catch (err) { if (err.message.indexOf('NOT NULL constraint failed:') === 0) { err.code = 'SQLITE_CONSTRAINT_NOTNULL' @@ -27,7 +27,7 @@ class WasmSqlite { throw err } }, - get: (params) => { + get(params) { const columns = stmt.getColumnNames() stmt.bind(params) stmt.step() @@ -38,7 +38,7 @@ class WasmSqlite { } return ret }, - all: (params) => { + all(params) { const columns = stmt.getColumnNames() const ret = [] stmt.bind(params) @@ -51,6 +51,12 @@ class WasmSqlite { ret.push(obj) } return ret + }, + *iterate(params) { + stmt.bind(params) + while (stmt.step()) { + yield stmt.get() + } } } this.gc.register(ret, stmt) diff --git a/sqlite/package.json b/sqlite/package.json index 5ca8845e5..c132dc7d6 100644 --- a/sqlite/package.json +++ b/sqlite/package.json @@ -29,11 +29,14 @@ "@cap-js/db-service": "^2", "better-sqlite3": "^12.0.0" }, - "optionalDependencies": { - "sql.js": "^1.10.3" - }, "peerDependencies": { - "@sap/cds": ">=9" + "@sap/cds": ">=9", + "sql.js": "^1.13.0" + }, + "peerDependenciesMeta": { + "sql.js": { + "optional": true + } }, "cds": { "requires": { From 54c4b27f27202c479ca5e433277db4e8df8c8c9f Mon Sep 17 00:00:00 2001 From: Bob den Os Date: Wed, 17 Sep 2025 11:31:56 +0200 Subject: [PATCH 9/9] Run sql.js tests after all other tests --- .github/workflows/sqlite-wasm.yml | 37 ------------------------------- .github/workflows/test.yml | 4 ++++ 2 files changed, 4 insertions(+), 37 deletions(-) delete mode 100644 .github/workflows/sqlite-wasm.yml diff --git a/.github/workflows/sqlite-wasm.yml b/.github/workflows/sqlite-wasm.yml deleted file mode 100644 index 1fca0ff55..000000000 --- a/.github/workflows/sqlite-wasm.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Tests WASM - -on: - push: - branches: [main] - pull_request: - types: [opened, synchronize, reopened, auto_merge_enabled] - -# Allow parallel jobs on `main`, so that each commit is tested. For PRs, run only the latest commit. -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 5 - name: Node.js ${{ matrix.node }} - - strategy: - fail-fast: true - matrix: - node: [18] - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - cache: 'npm' - - - run: npm ci - # Remove better-sqlite3 to force switching to sql.js - - run: npm install sql.js && rm -rf node_modules/better-sqlite3/ - - run: npm test -w sqlite -- --maxWorkers=1 - env: - FORCE_COLOR: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e09f33fa6..90e17fc2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,3 +45,7 @@ jobs: FORCE_COLOR: true TAG: ${{ steps.hxe.outputs.TAG }} IMAGE_ID: ${{ steps.hxe.outputs.IMAGE_ID }} + - run: npm install sql.js && rm -rf node_modules/better-sqlite3/ && npm test -w sqlite + env: + cds_features_pool: true + FORCE_COLOR: true