Loadable extensions release

.github/workflows/main.yml

@@ -134,6 +134,22 @@ jobs:
       - run: cargo build --features sqlcipher --workspace --all-targets --verbose
       - run: cargo test --features sqlcipher --workspace --all-targets --verbose
 
+  example-extension:
+    name: Test loadable_extension with example-extension
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: hecrj/setup-rust-action@v1
+      - run: ./example-extension/integration-test.sh
+
+  example-embedded-extension:
+    name: Test loadable_extension_embedded with example-embedded-extension
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: hecrj/setup-rust-action@v1
+      - run: ./example-embedded-extension/integration-test.sh
+
   sanitizer:
     name: Address Sanitizer
     runs-on: ubuntu-latest
@@ -170,6 +186,7 @@ jobs:
       - run: cargo clippy --all-targets --workspace --features bundled -- -D warnings
       # Clippy with all non-conflicting features
       - run: cargo clippy --all-targets --workspace --features 'bundled-full session buildtime_bindgen' -- -D warnings
+      - run: (cd example-extension && cargo clippy --all-targets -- -D warnings)
 
   # Ensure patch is formatted.
   fmt:

CONTRIBUTORS.md

@@ -0,0 +1,10 @@
+Contributors
+============
+
+"The rusqlite developers" referred to in the copyright notice in the LICENSE
+file is intended to include all individual developers who have contributed
+code to the rusqlite project.
+
+In addition, it includes the following organisations who retain copyright in
+portions of the code (this list is not intended to be comprehensive):
+ - Genomics plc

Cargo.toml

@@ -40,7 +40,8 @@ backup = ["libsqlite3-sys/min_sqlite_version_3_6_23"]
 blob = ["libsqlite3-sys/min_sqlite_version_3_7_7"]
 collation = []
 # sqlite3_create_function_v2: 3.7.3 (2010-10-08)
-functions = ["libsqlite3-sys/min_sqlite_version_3_7_7"]
+# functions requires ffi::SQLITE_DETERMINISTIC: 3.8.3 (2014-02-03)
+functions = ["modern_sqlite"]
 # sqlite3_log: 3.6.23 (2010-03-09)
 trace = ["libsqlite3-sys/min_sqlite_version_3_6_23"]
 # sqlite3_db_release_memory: 3.7.10 (2012-01-16)
@@ -49,6 +50,8 @@ bundled = ["libsqlite3-sys/bundled", "modern_sqlite"]
 bundled-sqlcipher = ["libsqlite3-sys/bundled-sqlcipher", "bundled"]
 bundled-sqlcipher-vendored-openssl = ["libsqlite3-sys/bundled-sqlcipher-vendored-openssl", "bundled-sqlcipher"]
 buildtime_bindgen = ["libsqlite3-sys/buildtime_bindgen"]
+loadable_extension = ["libsqlite3-sys/loadable_extension"]
+loadable_extension_embedded = ["loadable_extension", "libsqlite3-sys/loadable_extension_embedded"]
 limits = []
 hooks = []
 i128_blob = []
@@ -58,11 +61,11 @@ unlock_notify = ["libsqlite3-sys/unlock_notify"]
 vtab = ["libsqlite3-sys/min_sqlite_version_3_7_7"]
 csvtab = ["csv", "vtab"]
 # pointer passing interfaces: 3.20.0
-array = ["vtab"]
+array = ["modern_sqlite", "vtab"]
 # session extension: 3.13.0
-session = ["libsqlite3-sys/session", "hooks"]
+session = ["modern_sqlite", "libsqlite3-sys/session", "hooks"]
 # window functions: 3.25.0
-window = ["functions"]
+window = ["modern_sqlite", "functions"]
 # 3.9.0
 series = ["vtab"]
 # check for invalid query.

README.md

@@ -193,6 +193,7 @@ pregenerated bindings are chosen:
 * `min_sqlite_version_3_6_8` - SQLite 3.6.8 bindings (this is the default)
 * `min_sqlite_version_3_6_23` - SQLite 3.6.23 bindings
 * `min_sqlite_version_3_7_7` - SQLite 3.7.7 bindings
+* `min_sqlite_version_3_7_16` - SQLite 3.7.16 bindings
 
 If you use any of the `bundled` features, you will get pregenerated bindings for the
 bundled version of SQLite/SQLCipher. If you need other specific pregenerated binding

example-embedded-extension/.gitignore

@@ -0,0 +1,3 @@
+example-embedded-extension.h
+example-c-host-extension/libexample_c_host_extension.so
+target/

example-embedded-extension/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "example-embedded-extension"
+version = "0.0.1"
+authors = ["The rusqlite developers"]
+edition = "2018"
+repository = "https://github.com/rusqlite/rusqlite"
+description = "Example embedded extension to demonstrate and test rusqlite feature loadable_extension_embedded"
+license = "MIT"
+keywords = ["sqlite", "extension"]
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+
+[dependencies.rusqlite]
+path = ".."
+default-features = false
+features = ["loadable_extension_embedded", "vtab", "functions", "bundled"]
+
+[build-dependencies]
+cbindgen = "0.18.0"
+
+[workspace]
+members = []

example-embedded-extension/build.rs

@@ -0,0 +1,15 @@
+extern crate cbindgen;
+
+use std::env;
+
+fn main() {
+    let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+
+    // generate bindings to be included by the host extension that embeds us
+    cbindgen::Builder::new()
+        .with_language(cbindgen::Language::C)
+        .with_crate(crate_dir)
+        .generate()
+        .expect("Unable to generate bindings")
+        .write_to_file("example-embedded-extension.h");
+}

example-embedded-extension/example-c-host-extension/example_c_host_extension.c

@@ -0,0 +1,22 @@
+// include the sqlite3 extension header and call macros as documented in https://sqlite.org/loadext.html
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+
+// include the cbindgen-generated bindings for the embedded extension
+#include "example-embedded-extension.h"
+
+// the extension entry point
+int sqlite3_examplechostextension_init(
+                                        sqlite3 *db,
+                                        char **pzErrMsg,
+                                        const sqlite3_api_routines *pApi
+                                        ) {
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+
+  // for this example, we essentially just pass through to the embedded
+  // extension and return the result.
+  rc = example_embedded_extension_init(db, pzErrMsg);
+
+  return rc;
+}

example-embedded-extension/integration-test.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+# the crate dir is where this script is located
+crate_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# location of the cdylib embedded library within the target dir to be embedded within the c host extension
+example_embedded_extension_lib_dir="${crate_dir}/target/debug"
+example_embedded_extension_lib="example_embedded_extension"
+
+# location of the c host extension to be loaded by sqlite
+example_c_host_extension_dir="${crate_dir}/example-c-host-extension"
+example_c_host_extension="${example_c_host_extension_dir}/libexample_c_host_extension" # sqlite will try adding .so, .dll, .dylib to this on its own
+
+# expected output from vtable query
+expected_vtable_output="example_embedded_test_value"
+
+# expected output from function query
+expected_function_output="Example embedded extension loaded correctly!"
+
+# sqlite3 include dir (location of sqlite3ext.h) - can be set by SQLITE3_INCLUDE_DIR env var or defaults to bundled version
+sqlite3_include_dir=${SQLITE3_INCLUDE_DIR:-${crate_dir}/../libsqlite3-sys/sqlite3}
+
+>&2 echo "checking for sqlite3 shell"
+sqlite3_cmd=$(which sqlite3)
+>&2 echo "sqlite3 found: ${sqlite3_cmd}"
+
+# build the example-embedded-extension crate
+>&2 echo "building the example-embedded-extension crate in ${crate_dir}"
+(cd "${crate_dir}" && cargo build --all-targets --verbose)
+>&2 echo "successfully built the example-embedded-extension crate"
+
+# build the C-based host extension
+>&2 echo "building the example-c-host-extension"
+clang -g -fPIC -O2 -shared -I${sqlite3_include_dir} -I${crate_dir} -L${example_embedded_extension_lib_dir} -Wl,-rpath,${example_embedded_extension_lib_dir} -l${example_embedded_extension_lib} ${example_c_host_extension_dir}/example_c_host_extension.c -o ${example_c_host_extension}.so
+>&2 echo "successfully built the example-c-host-extension"
+
+>&2 echo "running sqlite3 (${sqlite3_cmd}) to test loadable_extension_embedded ${example_c_host_extension} vtable (embedded within C-based extension)"
+actual_vtable_output=$(${sqlite3_cmd} -cmd ".load ${example_c_host_extension}" :memory: "SELECT value FROM example_embedded LIMIT 1;")
+>&2 echo "sqlite3 command returned successfully from vtable test, checking output is as expected"
+test "${actual_vtable_output}" = "${expected_vtable_output}" && echo "OK" || (echo "vtable output '${actual_vtable_output}' was not as expected '${expected_vtable_output}'"; echo "FAIL"; exit 1)
+
+>&2 echo "running sqlite3 (${sqlite3_cmd}) to test loadable_extension_embedded ${example_c_host_extension} function (embedded within C-based extension)"
+actual_function_output=$(${sqlite3_cmd} -cmd ".load ${example_c_host_extension}" :memory: "SELECT example_embedded_test_function();")
+>&2 echo "sqlite3 command returned successfully from function test, checking output is as expected"
+test  "${actual_function_output}" = "${expected_function_output}" && echo "OK" || (echo "function output '${actual_function_output}' was not as expected '${expected_function_output}'"; echo "FAIL"; exit 1)
+
+>&2 echo "All tests passed."
+exit 0

example-embedded-extension/src/lib.rs

@@ -0,0 +1,135 @@
+use crate::ffi::loadable_extension_embedded_init; // required feature `loadable_extension_embedded`
+use std::marker::PhantomData;
+use std::os::raw::{c_char, c_int};
+
+use rusqlite::vtab::{
+    eponymous_only_module, sqlite3_vtab, sqlite3_vtab_cursor, Context, IndexInfo, VTab,
+    VTabConnection, VTabCursor, Values,
+};
+use rusqlite::{
+    ffi,
+    functions::FunctionFlags,
+    types::{ToSqlOutput, Value},
+};
+use rusqlite::{to_sqlite_error, Connection, Result};
+
+/// example_embedded_extension_init is the entry point for this library.
+///
+/// This crate produces a cdylib that is intended to be embedded within
+/// (i.e. linked into) another library that implements the sqlite loadable
+/// extension entrypoint.
+///
+/// In the case of this example code, refer to the `example-c-host-extension`
+/// C code to find where this entry point is invoked.
+///
+/// Note that this interface is private between the host extension and this
+/// library - it can have any signature as long as it passes the *sqlite3 db
+/// pointer so we can use it to initialize our rusqlite::Connection.
+///
+/// It does *not* have to return sqlite status codes (such as SQLITE_OK), we
+/// just do that here to keep the C extension simple.
+///
+/// # Safety
+///
+/// The C host extension must pass a pointer to a valid `sqlite3` struct in
+/// `db`` and either null or a pointer to a char* in `pz_err_msg`.
+#[no_mangle]
+pub unsafe extern "C" fn example_embedded_extension_init(
+    db: *mut ffi::sqlite3,
+    pz_err_msg: *mut *mut c_char,
+) -> c_int {
+    loadable_extension_embedded_init();
+
+    let res = example_embedded_init(db);
+    if let Err(err) = res {
+        return to_sqlite_error(&err, pz_err_msg);
+    }
+
+    ffi::SQLITE_OK
+}
+
+#[repr(C)]
+struct ExampleEmbeddedTab {
+    /// Base class. Must be first
+    base: sqlite3_vtab,
+}
+
+unsafe impl<'vtab> VTab<'vtab> for ExampleEmbeddedTab {
+    type Aux = ();
+    type Cursor = ExampleEmbeddedTabCursor<'vtab>;
+
+    fn connect(
+        _: &mut VTabConnection,
+        _aux: Option<&()>,
+        _args: &[&[u8]],
+    ) -> Result<(String, ExampleEmbeddedTab)> {
+        let vtab = ExampleEmbeddedTab {
+            base: sqlite3_vtab::default(),
+        };
+        Ok(("CREATE TABLE x(value TEXT)".to_owned(), vtab))
+    }
+
+    fn best_index(&self, info: &mut IndexInfo) -> Result<()> {
+        info.set_estimated_cost(1.);
+        Ok(())
+    }
+
+    fn open(&'vtab self) -> Result<ExampleEmbeddedTabCursor<'vtab>> {
+        Ok(ExampleEmbeddedTabCursor::default())
+    }
+}
+
+#[derive(Default)]
+#[repr(C)]
+struct ExampleEmbeddedTabCursor<'vtab> {
+    /// Base class. Must be first
+    base: sqlite3_vtab_cursor,
+    /// The rowid
+    row_id: i64,
+    phantom: PhantomData<&'vtab ExampleEmbeddedTab>,
+}
+
+unsafe impl VTabCursor for ExampleEmbeddedTabCursor<'_> {
+    fn filter(
+        &mut self,
+        _idx_num: c_int,
+        _idx_str: Option<&str>,
+        _args: &Values<'_>,
+    ) -> Result<()> {
+        self.row_id = 1;
+        Ok(())
+    }
+
+    fn next(&mut self) -> Result<()> {
+        self.row_id += 1;
+        Ok(())
+    }
+
+    fn eof(&self) -> bool {
+        self.row_id > 1
+    }
+
+    fn column(&self, ctx: &mut Context, _: c_int) -> Result<()> {
+        ctx.set_result(&"example_embedded_test_value".to_string())
+    }
+
+    fn rowid(&self) -> Result<i64> {
+        Ok(self.row_id)
+    }
+}
+
+fn example_embedded_init(db: *mut ffi::sqlite3) -> Result<()> {
+    let conn = unsafe { Connection::from_handle(db)? };
+    eprintln!("inited example embedded extension module {:?}", db);
+    conn.create_scalar_function(
+        "example_embedded_test_function",
+        0,
+        FunctionFlags::SQLITE_DETERMINISTIC,
+        |_ctx| {
+            Ok(ToSqlOutput::Owned(Value::Text(
+                "Example embedded extension loaded correctly!".to_string(),
+            )))
+        },
+    )?;
+    conn.create_module::<ExampleEmbeddedTab>("example_embedded", eponymous_only_module::<ExampleEmbeddedTab>(), None)
+}

example-extension/.gitignore

@@ -0,0 +1,3 @@
+/target/
+/doc/
+Cargo.lock

example-extension/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "example-extension"
+version = "0.0.1"
+authors = ["The rusqlite developers"]
+edition = "2018"
+repository = "https://github.com/rusqlite/rusqlite"
+description = "Example extension to demonstrate and test rusqlite feature loadable_extension"
+license = "MIT"
+keywords = ["sqlite", "extension"]
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+
+[dependencies.rusqlite]
+path = ".."
+default-features = false
+features = ["loadable_extension", "vtab", "functions", "bundled"]
+
+[workspace]
+members = []