Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,57 @@
## [Unreleased]

### Fixed

- **SQLite on the phone has FTS5 and exqlite's other features (MOB-487).**
mob compiled exqlite's SQLite amalgamation with only
`-DSQLITE_THREADSAFE=1`, while the host NIF (built by exqlite's Makefile)
also enables FTS3/4/5, R*Tree, Geopoly, math functions, RBU and dbstat. A
query on an FTS5 table passed host tests and failed on the device with
`no such module: fts5`. The iOS simulator NIF, the iOS device
`sqlite3_nif.a` and `mix mob.release --ios` now compile with exqlite's
defines, from one list, `MobDev.Exqlite.cflags/0`; a test pins it against
the locked exqlite's Makefile and against mob_new's Android templates.
Note that exqlite's set includes `SQLITE_DQS=0`, so the device, like the
host, now rejects double-quoted string literals (`WHERE x = "a"`).

### Upgrading

- **MOB-487, Android.** iOS picks the defines up from mob_dev on the next
`mix mob.deploy --native`. Android compiles SQLite in the app's own
`android/app/src/main/jni/build.zig` (and `CMakeLists.txt`, the fallback
when zig didn't run), so existing apps edit those (mob_new's templates
carry it for new apps). In `build.zig`, in `const sqlite_flags`, replace
the line `"-DSQLITE_THREADSAFE=1",` with:

```zig
"-DSQLITE_THREADSAFE=1",
"-DSQLITE_USE_URI=1",
"-DSQLITE_LIKE_DOESNT_MATCH_BLOBS=1",
"-DSQLITE_DQS=0",
"-DHAVE_USLEEP=1",
"-DALLOW_COVERING_INDEX_SCAN=1",
"-DENABLE_FTS3_PARENTHESIS=1",
"-DENABLE_LOAD_EXTENSION=1",
"-DENABLE_SOUNDEX=1",
"-DENABLE_STAT4=1",
"-DENABLE_UPDATE_DELETE_LIMIT=1",
"-DSQLITE_ENABLE_FTS3=1",
"-DSQLITE_ENABLE_FTS4=1",
"-DSQLITE_ENABLE_FTS5=1",
"-DSQLITE_ENABLE_GEOPOLY=1",
"-DSQLITE_ENABLE_MATH_FUNCTIONS=1",
"-DSQLITE_ENABLE_RBU=1",
"-DSQLITE_ENABLE_RTREE=1",
"-DSQLITE_OMIT_DEPRECATED=1",
"-DSQLITE_ENABLE_DBSTAT_VTAB=1",
```

In `CMakeLists.txt`, in `target_compile_definitions(sqlite3_nif PRIVATE`,
replace `SQLITE_THREADSAFE=1` with the same names without `-D` and quotes
(`SQLITE_THREADSAFE=1`, `SQLITE_USE_URI=1`, … one per line), and add `m`
after `log` in `target_link_libraries(sqlite3_nif`. Then run
`mix mob.deploy --native`. mob_new's CHANGELOG has the CMake lines in full.

## [0.7.28] - 2026-10-15

### Fixed
Expand Down
51 changes: 51 additions & 0 deletions lib/mob_dev/exqlite.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
defmodule MobDev.Exqlite do
@moduledoc """
The compile-time defines mob builds exqlite's SQLite amalgamation with.

On the host, exqlite compiles `sqlite3.c` with its own Makefile, which
enables FTS3/4/5, R*Tree, Geopoly, math functions and more. On a phone,
mob compiles the same amalgamation itself (the iOS simulator `.so`, the
iOS device `.a`, `mix mob.release --ios`), and the generated Android
`build.zig` / `CMakeLists.txt` do the same. Every one of those builds uses
this set, so the device SQLite has the features host tests rely on: an
FTS5 table that works under `mix test` also works on the phone (MOB-487).

The list is exqlite's Makefile `CFLAGS += -D…` lines, verbatim and in
order, including the unprefixed ones (`ENABLE_STAT4`, …) that SQLite
ignores; `MobDev.ExqliteTest` pins it against the locked exqlite's
Makefile. mob_new's Android templates carry the same lines (it is a
self-contained archive and can't call this module); the same test keeps
them in sync.
"""

@defines [
"SQLITE_THREADSAFE=1",
"SQLITE_USE_URI=1",
"SQLITE_LIKE_DOESNT_MATCH_BLOBS=1",
"SQLITE_DQS=0",
"HAVE_USLEEP=1",
"ALLOW_COVERING_INDEX_SCAN=1",
"ENABLE_FTS3_PARENTHESIS=1",
"ENABLE_LOAD_EXTENSION=1",
"ENABLE_SOUNDEX=1",
"ENABLE_STAT4=1",
"ENABLE_UPDATE_DELETE_LIMIT=1",
"SQLITE_ENABLE_FTS3=1",
"SQLITE_ENABLE_FTS4=1",
"SQLITE_ENABLE_FTS5=1",
"SQLITE_ENABLE_GEOPOLY=1",
"SQLITE_ENABLE_MATH_FUNCTIONS=1",
"SQLITE_ENABLE_RBU=1",
"SQLITE_ENABLE_RTREE=1",
"SQLITE_OMIT_DEPRECATED=1",
"SQLITE_ENABLE_DBSTAT_VTAB=1"
]

@doc "The defines, as `NAME=VALUE` strings."
@spec defines() :: [String.t()]
def defines, do: @defines

@doc "The defines as compiler flags (`-DNAME=VALUE`)."
@spec cflags() :: [String.t()]
def cflags, do: Enum.map(@defines, &("-D" <> &1))
end
70 changes: 37 additions & 33 deletions lib/mob_dev/native_build.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2250,34 +2250,37 @@ defmodule MobDev.NativeBuild do
# The macOS-compiled NIF from `mix deps.compile` is incompatible with
# the iOS sim (wrong platform tag). Recompile against iphonesimulator
# SDK so dlopen succeeds inside the simulator process.
args = [
"-sdk",
"iphonesimulator",
"cc",
"-arch",
"arm64",
"-mios-simulator-version-min=17.0",
"-isysroot",
sdkroot,
"-Os",
"-ffunction-sections",
"-fdata-sections",
"-dynamiclib",
"-undefined",
"dynamic_lookup",
"-I",
"deps/exqlite/c_src",
"-I",
"#{otp_root}/#{erts_vsn}/include",
"-I",
"#{otp_root}/#{erts_vsn}/include/aarch64-apple-iossimulator",
"-DSQLITE_THREADSAFE=1",
"-Wno-#warnings",
"deps/exqlite/c_src/sqlite3_nif.c",
"deps/exqlite/c_src/sqlite3.c",
"-o",
out_so
]
args =
[
"-sdk",
"iphonesimulator",
"cc",
"-arch",
"arm64",
"-mios-simulator-version-min=17.0",
"-isysroot",
sdkroot,
"-Os",
"-ffunction-sections",
"-fdata-sections",
"-dynamiclib",
"-undefined",
"dynamic_lookup",
"-I",
"deps/exqlite/c_src",
"-I",
"#{otp_root}/#{erts_vsn}/include",
"-I",
"#{otp_root}/#{erts_vsn}/include/aarch64-apple-iossimulator"
] ++
MobDev.Exqlite.cflags() ++
[
"-Wno-#warnings",
"deps/exqlite/c_src/sqlite3_nif.c",
"deps/exqlite/c_src/sqlite3.c",
"-o",
out_so
]

case System.cmd("xcrun", args, stderr_to_stdout: true, into: IO.stream()) do
{_, 0} -> :ok
Expand Down Expand Up @@ -3503,8 +3506,10 @@ defmodule MobDev.NativeBuild do
"-I",
"#{otp_root}/#{erts_vsn}/include",
"-I",
"#{otp_root}/#{erts_vsn}/include/internal",
"-DSQLITE_THREADSAFE=1",
"#{otp_root}/#{erts_vsn}/include/internal"
] ++
MobDev.Exqlite.cflags() ++
[
"-DSTATIC_ERLANG_NIF_LIBNAME=sqlite3_nif",
"-Wno-#warnings",
"-c",
Expand All @@ -3516,10 +3521,9 @@ defmodule MobDev.NativeBuild do
:ok <-
run_cc(
common_cc ++
["-I", "deps/exqlite/c_src"] ++
MobDev.Exqlite.cflags() ++
[
"-I",
"deps/exqlite/c_src",
"-DSQLITE_THREADSAFE=1",
"-Wno-#warnings",
"-c",
"deps/exqlite/c_src/sqlite3.c",
Expand Down
6 changes: 4 additions & 2 deletions lib/mob_dev/release.ex
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,8 @@ defmodule MobDev.Release do
OUTPUT_DIR="${MOB_RELEASE_OUTPUT_DIR:?MOB_RELEASE_OUTPUT_DIR not set}"
""" <>
release_scratch_sh() <>
"# exqlite's SQLite defines (MobDev.Exqlite, MOB-487)\n" <>
"SQLITE_CFLAGS=\"#{Enum.join(MobDev.Exqlite.cflags(), " ")}\"\n" <>
~S"""

ERTS_VSN=$(ls "$OTP_ROOT" | grep '^erts-' | sort -V | tail -1)
Expand Down Expand Up @@ -779,10 +781,10 @@ defmodule MobDev.Release do
mkdir -p "$BUILD_DIR_TMP"
$CC -I "$EXQLITE_SRC" -I "$OTP_ROOT/$ERTS_VSN/include" \
-I "$OTP_ROOT/$ERTS_VSN/include/internal" \
-DSQLITE_THREADSAFE=1 -DSTATIC_ERLANG_NIF_LIBNAME=sqlite3_nif \
$SQLITE_CFLAGS -DSTATIC_ERLANG_NIF_LIBNAME=sqlite3_nif \
-Wno-\#warnings \
-c "$EXQLITE_SRC/sqlite3_nif.c" -o "$BUILD_DIR_TMP/sqlite3_nif.o"
$CC -I "$EXQLITE_SRC" -DSQLITE_THREADSAFE=1 -Wno-\#warnings \
$CC -I "$EXQLITE_SRC" $SQLITE_CFLAGS -Wno-\#warnings \
-c "$EXQLITE_SRC/sqlite3.c" -o "$BUILD_DIR_TMP/sqlite3.o"
$(xcrun -find ar) rcs "$EXQLITE_LIB_DIR/priv/sqlite3_nif.a" \
"$BUILD_DIR_TMP/sqlite3_nif.o" "$BUILD_DIR_TMP/sqlite3.o"
Expand Down
4 changes: 4 additions & 0 deletions mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,10 @@ defmodule MobDev.MixProject do
# Lets us test "given inputs, clang is invoked with these args"
# without actually running clang.
{:mox, "~> 1.2", only: :test},
# exqlite's source, never compiled or loaded: MobDev.ExqliteTest reads
# its Makefile to pin MobDev.Exqlite's SQLite defines (MOB-487). Its
# children still resolve, which adds db_connection to the test build.
{:exqlite, "~> 0.42", only: :test, runtime: false, compile: false, app: false},
{:erlfmt, "~> 1.8", only: :dev, runtime: false},
# Dev-only dead-code detector. Wires in via the `:unused` compiler
# tracer; ignore list maintained inline below since this codebase has
Expand Down
2 changes: 2 additions & 0 deletions mix.lock
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
"cc_precompiler": {:hex, :cc_precompiler, "0.1.11", "8c844d0b9fb98a3edea067f94f616b3f6b29b959b6b3bf25fee94ffe34364768", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "3427232caf0835f94680e5bcf082408a70b48ad68a5f5c0b02a3bea9f3a075b9"},
"credo": {:hex, :credo, "1.7.18", "5c5596bf7aedf9c8c227f13272ac499fe8eae6237bd326f2f07dfc173786f042", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a189d164685fd945809e862fe76a7420c4398fa288d76257662aecb909d6b3e5"},
"db_connection": {:hex, :db_connection, "2.10.2", "ae391e803a5adff104da913c2fc1c0c14a37f8b10001dcef568796e1fb7bf95c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "510b14482330f1af6490a2fa0efd8d4f1435d1529b165647df22ac0f2df0fa93"},
"earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"},
"elixir_make": {:hex, :elixir_make, "0.10.0", "16577e2583a79bb79237bbff349619ef5d80afffc07eac6e4faf0d00e2ddaf7d", [:mix], [], "hexpm", "dc1f09fb7fa68866b886abd5f0f3c83553b1a19a52359a899e92af1bb3b31982"},
"eqrcode": {:hex, :eqrcode, "0.2.1", "d12838813e8fc87b8940cc05f9baadb189031f6009facdc56ff074375ec73b6e", [:mix], [], "hexpm", "d5828a222b904c68360e7dc2a40c3ef33a1328b7c074583898040f389f928025"},
Expand All @@ -12,6 +13,7 @@
"ex_dna": {:hex, :ex_dna, "1.5.2", "486aff9dea0790df17041665abdf9b36f444bcf764fb0a268ea4101cfee58db9", [:mix], [{:credo, "~> 1.7", [hex: :credo, repo: "hexpm", optional: true]}, {:gen_lsp, "~> 0.11", [hex: :gen_lsp, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: true]}], "hexpm", "29f9935fd27b12a33411168ca9a38b5cc5ef7e8ef91061654731486a1b85f170"},
"ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"},
"ex_slop": {:hex, :ex_slop, "0.4.2", "142aba9a82eddfb258e39c45d59392ab3cdb6b5a3ad401b09b362b7134fc54eb", [:mix], [{:credo, "~> 1.7", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "c7f5316f755f83566e7a0a049f6fedfcd5ff916fce83c6ebfdf806be62fd7a69"},
"exqlite": {:hex, :exqlite, "0.42.0", "3effa63f9fa365893fc318c597a9dc1253aa5acf1abd138660efacf830317f6d", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "01e1cce89dd6be4966acc87d426d7ba93455820138fe1564f422a6478419f1f7"},
"file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"},
"finch": {:hex, :finch, "0.23.0", "e3f9287ac25a8832f848b144c2b57346aac65b205e2e0629a52adfe6507fd837", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "80e58d3f936f57e3fdf404f83a3642897ae6d9fb642934e46da4d8fe761b99d5"},
"glob_ex": {:hex, :glob_ex, "0.1.12", "7b2d9369c20e2697efcfd185d13d6e84c94cd3bfd2730fbde613141c2e015c00", [:mix], [], "hexpm", "2e2fac83f113514434c7eaf267b4c38af2f91766f1cab2c5db7053b7fc1ee0bb"},
Expand Down
58 changes: 58 additions & 0 deletions test/mob_dev/exqlite_test.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
defmodule MobDev.ExqliteTest do
use ExUnit.Case, async: true

# MOB-487: device builds compiled sqlite3.c with only -DSQLITE_THREADSAFE=1,
# so an FTS5 table that worked under host tests failed on the phone with
# `no such module: fts5`. Every mob build of the amalgamation must use
# exqlite's own defines, the ones its Makefile compiles the host NIF with.

alias MobDev.Exqlite

test "the define set enables FTS5" do
assert "-DSQLITE_ENABLE_FTS5=1" in Exqlite.cflags()
end

test "the define set is exqlite's Makefile set, in order" do
makefile = Path.join(Mix.Project.deps_paths().exqlite, "Makefile")
line = Regex.compile!(~S{^CFLAGS \+= -D(\S+)$}, "m")
makefile_defines = Regex.scan(line, File.read!(makefile), capture: :all_but_first)

assert Exqlite.defines() == List.flatten(makefile_defines)
end

# mob_new is a self-contained archive and can't call MobDev.Exqlite, so its
# Android templates carry the list themselves. Read them from the mob_new
# checkout CI provides in MOB_NEW_DIR: an installed mob_new archive or a
# stale ~/code/mob_new (MobDev.Adopt.Generator.templates_root/1's other
# sources) can predate the list.
if is_nil(System.get_env("MOB_NEW_DIR")),
do: @tag(skip: "set MOB_NEW_DIR to a mob_new checkout")

test "mob_new's Android build.zig and CMakeLists.txt compile SQLite with the same set" do
jni =
Path.join(
System.fetch_env!("MOB_NEW_DIR"),
"priv/templates/mob.new/android/app/src/main/jni"
)

zig_define = Regex.compile!(~S{^"-D(.+)",$})

zig =
jni
|> Path.join("build.zig.eex")
|> define_block()
|> Enum.map(&(Regex.run(zig_define, &1, capture: :all_but_first) |> hd()))

cmake = jni |> Path.join("CMakeLists.txt.eex") |> define_block()

assert {zig, cmake} == {Exqlite.defines(), Exqlite.defines()}
end

# The lines between the `exqlite SQLite defines (begin)` and `(end)` marker
# comments (`//` in build.zig, `#` in CMakeLists.txt), trimmed.
defp define_block(path) do
marker = Regex.compile!(~S{^\s*(?://|#) exqlite SQLite defines \((?:begin|end)\)$}, "m")
[_, block, _] = Regex.split(marker, File.read!(path))
block |> String.split("\n", trim: true) |> Enum.map(&String.trim/1)
end
end
Loading