diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c90ad7..da37e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - changed: Update sqlite to `3.53.4`. +- added: `EXQLITE_EXTRA_SRC` build variable to statically compile SQLite extensions (e.g. sqlite-vec) into the NIF, for targets where `load_extension` is unavailable (iOS) or undesirable. ## v0.39.0 diff --git a/Makefile b/Makefile index c750b91..3349d56 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,20 @@ ARCHIVE_NAME = $(PREFIX)/sqlite3_nif.a OBJ = $(SRC:c_src/%.c=$(BUILD)/%.o) +# Statically compile additional C sources (for example SQLite extensions such +# as sqlite-vec or spellfix1) into the NIF. Space-separated list of file paths. +# Pair with EXQLITE_SYSTEM_CFLAGS to define SQLITE_EXTRA_INIT so the extension +# registers itself on every connection - see "Statically compiling SQLite3 +# extensions" in the README. Only meaningful when compiling the bundled SQLite +# (ignored under EXQLITE_USE_SYSTEM, where linking is the system library's job). +ifneq ($(EXQLITE_EXTRA_SRC),) +ifeq ($(EXQLITE_USE_SYSTEM),) + EXTRA_OBJ = $(addprefix $(BUILD)/extra_,$(notdir $(EXQLITE_EXTRA_SRC:.c=.o))) + OBJ += $(EXTRA_OBJ) + vpath %.c $(sort $(dir $(EXQLITE_EXTRA_SRC))) +endif +endif + ifneq ($(CROSSCOMPILE),) ifeq ($(CROSSCOMPILE), Android) CFLAGS:=$(filter-out -O2,$(CFLAGS)) @@ -144,6 +158,10 @@ $(BUILD)/%.o: c_src/%.c @echo " CC $(notdir $@)" $(CC) -c $(ERL_CFLAGS) $(CFLAGS) -MMD -MP -o $@ $< +$(BUILD)/extra_%.o: %.c + @echo " CC $(notdir $@)" + $(CC) -c $(ERL_CFLAGS) $(CFLAGS) -MMD -MP -o $@ $< + # Include dependency files for automatic header tracking -include $(OBJ:.o=.d) diff --git a/README.md b/README.md index 2b9d433..72ba711 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,55 @@ See [Exqlite.Connection.connect/1](https://hexdocs.pm/exqlite/Exqlite.Connection for more information. When using extensions for SQLite3, they must be compiled for the environment you are targeting. +### Statically compiling SQLite3 extensions + +Run-time loading covers most cases, but some targets forbid `load_extension` +entirely (iOS refuses dynamic code loading; some hardened Linux deployments +disable it), and shipping one `.so` per architecture is its own maintenance +burden. For those cases the build supports compiling extension sources +directly into the NIF: + +```elixir +config :exqlite, + force_build: true, + make_env: %{ + "EXQLITE_EXTRA_SRC" => "/path/to/sqlite-vec.c /path/to/vec_init_shim.c", + "EXQLITE_SYSTEM_CFLAGS" => "-DSQLITE_CORE=1 -DSQLITE_EXTRA_INIT=exqlite_vec_init" + } +``` + +where the shim registers the extension for every future connection via +[`SQLITE_EXTRA_INIT`](https://sqlite.org/compile.html#extra_init) and +`sqlite3_auto_extension`: + +```c +/* vec_init_shim.c */ +#include "sqlite3.h" +#include "sqlite-vec.h" + +int exqlite_vec_init(const char *unused) { + (void)unused; + return sqlite3_auto_extension((void (*)(void))sqlite3_vec_init); +} +``` + +```elixir +{:ok, conn} = Exqlite.Sqlite3.open(":memory:") +{:ok, stmt} = Exqlite.Sqlite3.prepare(conn, "SELECT vec_distance_cosine(vec_f32('[1,0]'), vec_f32('[0,1]'))") +{:row, [1.0]} = Exqlite.Sqlite3.step(conn, stmt) +``` + +Notes: + +- `-DSQLITE_CORE=1` makes the extension compile against the core API instead + of the loadable-extension shim. +- Only applies when compiling the bundled SQLite (it is ignored under + `EXQLITE_USE_SYSTEM`), and requires `force_build: true` so a precompiled + NIF is not fetched instead. +- Extra sources are looked up by basename, so two files with the same + basename in different directories will collide. +- POSIX make only for now (`Makefile.win` does not implement it). + ## Why SQLite3 I needed an Ecto3 adapter to store time series data for a personal project. I