diff --git a/Dockerfile-multigres b/Dockerfile-multigres index 5184938fc4..6fe3acdcf8 100644 --- a/Dockerfile-multigres +++ b/Dockerfile-multigres @@ -264,13 +264,11 @@ RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' \ /etc/postgresql-custom/supautils.conf # Add orioledb to shared_preload_libraries and configure as default table access method -# Rewind: 1200s window, 100k transactions, 1280 x 8KB = 10MB buffer +# orioledb-17 is pinned to 17.11+, which added output_plugin_libraries +# as an allow-list for logical decoding output plugins. RUN sed -i "s/\(shared_preload_libraries.*\)'/\1, orioledb'/" /etc/postgresql/postgresql.conf && \ echo "default_table_access_method = 'orioledb'" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.enable_rewind = true" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.rewind_max_time = 1200" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.rewind_max_transactions = 100000" >> /etc/postgresql/postgresql.conf && \ - echo "orioledb.rewind_buffers = 1280" >> /etc/postgresql/postgresql.conf + echo "output_plugin_libraries = 'pgoutput, test_decoding, wal2json'" >> /etc/postgresql/postgresql.conf # Register orioledb before initdb migrations run RUN echo "CREATE EXTENSION orioledb;" > /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql && \ diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index 7cd70a3710..4405cc22b5 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -158,6 +158,10 @@ RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" && \ RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf" && \ echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf" +# orioledb-17 is pinned to 17.11+, which added output_plugin_libraries as an +# allow-list for logical decoding output plugins +RUN echo "output_plugin_libraries = 'pgoutput, test_decoding, wal2json'" >> "/etc/postgresql/postgresql.conf" + # Include schema migrations COPY migrations/db /docker-entrypoint-initdb.d/ COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index d4823238a7..7b230a8e61 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -48,6 +48,14 @@ path: '/etc/postgresql/postgresql.conf' state: 'present' + # orioledb-17 is pinned to 17.11+, which added output_plugin_libraries + # as an allow-list for logical decoding output plugins. + - name: Allow wal2json as a logical decoding output plugin + ansible.builtin.lineinfile: + line: "output_plugin_libraries = 'pgoutput, test_decoding, wal2json'" + path: '/etc/postgresql/postgresql.conf' + state: 'present' + - name: Add ORIOLEDB_ENABLED environment variable ansible.builtin.lineinfile: line: 'ORIOLEDB_ENABLED=true' diff --git a/ansible/vars.yml b/ansible/vars.yml index 097125f822..bcd8bc1853 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -7,9 +7,9 @@ postgres_major: - "17" - orioledb-17 postgres_release: - postgres15: "15.14.1.176" - postgres17: "17.6.1.176" - postgresorioledb-17: "17.9.0.029-orioledb" + postgresorioledb-17: "17.11.0.001-orioledb" + postgres17: "17.6.1.177" + postgres15: "15.14.1.177" supabase_admin_agent_splay: 30s ############################################################################################################### # The following block of yaml is for get_url and co throughout the playbook # diff --git a/docker/pgctld/orioledb-postgresql.conf.tmpl b/docker/pgctld/orioledb-postgresql.conf.tmpl index caedde706b..f307e4893f 100644 --- a/docker/pgctld/orioledb-postgresql.conf.tmpl +++ b/docker/pgctld/orioledb-postgresql.conf.tmpl @@ -298,7 +298,7 @@ wal_log_hints = 'on' #------------------------------------------------------------------------------ shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, auto_explain, pg_tle, plan_filter, supabase_vault, orioledb' default_table_access_method = 'orioledb' -orioledb.enable_rewind = true -orioledb.rewind_max_time = 1200 -orioledb.rewind_max_transactions = 100000 -orioledb.rewind_buffers = 1280 +# orioledb-17 is pinned to 17.11+, which added output_plugin_libraries +# as an allow-list for logical decoding output plugins. +output_plugin_libraries = 'pgoutput, test_decoding, wal2json' + diff --git a/migrations/schema-orioledb-17.sql b/migrations/schema-orioledb-17.sql index c1617fdc60..d4fb8c6198 100644 --- a/migrations/schema-orioledb-17.sql +++ b/migrations/schema-orioledb-17.sql @@ -4,8 +4,8 @@ \restrict SupabaseTestDumpKey123 --- Dumped from database version 17.9 --- Dumped by pg_dump version 17.9 +-- Dumped from database version 17.11 +-- Dumped by pg_dump version 17.11 SET statement_timeout = 0; SET lock_timeout = 0; diff --git a/nix/checks.nix b/nix/checks.nix index 118115e85f..518aac5690 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -379,6 +379,9 @@ # Add orioledb to shared_preload_libraries perl -pi -e "s/(shared_preload_libraries = ')/\$1orioledb, /" "$PGTAP_CLUSTER/postgresql.conf" log info "OrioleDB added to shared_preload_libraries" + # orioledb-17 is pinned to 17.11+, which added output_plugin_libraries + # as an allow-list for logical decoding output plugins. + echo "output_plugin_libraries = 'pgoutput, test_decoding, wal2json'" >> "$PGTAP_CLUSTER/postgresql.conf" fi # Check if postgresql.conf exists diff --git a/nix/config.nix b/nix/config.nix index 7ed5338e0a..a70fb4d617 100644 --- a/nix/config.nix +++ b/nix/config.nix @@ -56,8 +56,8 @@ in }; orioledb = { "17" = { - version = "17_20"; - hash = "sha256-HDrHTx9yeIPJoyIBs+BdAhBQqt1IEtQrG9pFfvHJqdg="; + version = "17_22"; + hash = "sha256-B3ibVCa7dSdrrR6VyhuJ36Hp/lqmxmH5gauZpc9ylkA="; }; }; }; diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 3c7558919e..c0fa3a5efe 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -15,10 +15,10 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "beta16"; - sha256 = "sha256-HCfNzMPt80nGeVwlstUCeMpdNZYd9KhLLHYyD/Hvuhk="; + rev = "beta18"; + sha256 = "sha256-kmNfneISVhlD9Pmnl69pkiAX77MzpzyJQB/5wQ1wiZc="; }; - version = "beta16"; + version = "beta18"; buildInputs = [ curl libkrb5 @@ -27,7 +27,7 @@ stdenv.mkDerivation rec { openssl ]; buildPhase = '' - make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=20 + make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=22 ''; separateDebugInfo = true; installPhase = '' diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index f0d8e37d46..1404839fd8 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -163,6 +163,9 @@ let ] ++ (lib.toList ((installedExtension "17").defaultSettings.shared_preload_libraries or [ ])); default_table_access_method = "orioledb"; + # orioledb-17 is pinned to 17.11+, which added output_plugin_libraries + # as an allow-list for logical decoding output plugins. + output_plugin_libraries = "pgoutput, test_decoding, wal2json"; } ); initdbArgs = [ diff --git a/nix/ext/tests/orioledb-rewind.nix b/nix/ext/tests/orioledb-rewind.nix deleted file mode 100644 index bb951f907f..0000000000 --- a/nix/ext/tests/orioledb-rewind.nix +++ /dev/null @@ -1,205 +0,0 @@ -{ self, pkgs }: -let - pname = "orioledb-rewind"; - testLib = import ./lib.nix { inherit self pkgs; }; -in -pkgs.testers.runNixOSTest { - name = pname; - nodes.server = - { ... }: - { - imports = [ - (testLib.makeSupabaseTestConfig { - majorVersion = "15"; - }) - ]; - - specialisation.orioledb17.configuration = testLib.makeOrioledbSpecialisation { - extraConfig = '' - orioledb.enable_rewind = true - orioledb.main_buffers = 1280 - orioledb.rewind_max_time = 1200 - orioledb.rewind_max_transactions = 100000 - ''; - }; - }; - testScript = - { nodes, ... }: - let - orioledb17-configuration = "${nodes.server.system.build.toplevel}/specialisation/orioledb17"; - in - '' - import time - - orioledb17_configuration = "${orioledb17-configuration}" - - start_all() - - # Wait for full Supabase initialization on PG 15 - server.wait_for_unit("supabase-db-init.service") - - with subtest("Switch to OrioleDB and show rewind config"): - server.execute( - f"{orioledb17_configuration}/bin/switch-to-configuration test >&2" - ) - server.wait_for_unit("supabase-db-init.service") - - # Verify OrioleDB is running - installed_extensions = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT extname FROM pg_extension WHERE extname = 'orioledb';\"" - ).strip() - assert "orioledb" in installed_extensions, ( - f"Expected orioledb extension to be installed, got: {installed_extensions}" - ) - - # Show all rewind-related settings - rewind_settings = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT name || ' = ' || setting FROM pg_settings WHERE name LIKE 'orioledb.%rewind%' OR name LIKE 'orioledb.%main_buffers%' ORDER BY name;\"" - ).strip() - print(f"OrioleDB rewind settings:\n{rewind_settings}") - - # Assert rewind is enabled - rewind_enabled = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SHOW orioledb.enable_rewind;\"" - ).strip() - assert rewind_enabled == "on", ( - f"Expected orioledb.enable_rewind = on, got: {rewind_enabled}" - ) - - # Print rewind queue/evicted lengths - queue_len = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT orioledb_get_rewind_queue_length();\"" - ).strip() - evicted_len = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT orioledb_get_rewind_evicted_length();\"" - ).strip() - print(f"Initial rewind queue length: {queue_len}") - print(f"Initial rewind evicted length: {evicted_len}") - - with subtest("Basic rewind"): - # Phase 1: Setup — matches bash script exactly - server.succeed( - "psql -U supabase_admin -d postgres -c \"DROP TABLE IF EXISTS rewind_test; CREATE TABLE rewind_test(x serial) USING orioledb; INSERT INTO rewind_test SELECT FROM generate_series(1, 100);\"" - ) - - # Capture xid, oxid, and hash in a single query — matches bash script - ids = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT pg_current_xact_id()::text::int4, orioledb_get_current_oxid()::int8, md5(string_agg(x::text, '.' ORDER BY x)) FROM rewind_test;\"" - ).strip() - parts = ids.split("|") - xid = parts[0] - oxid = parts[1] - pre_hash = parts[2] - print(f"Checkpoint: xid={xid} oxid={oxid} hash={pre_hash}") - - # Phase 2: Dirty the state — matches bash script - server.succeed( - "psql -U supabase_admin -d postgres -c \"INSERT INTO rewind_test SELECT FROM generate_series(1, 10); INSERT INTO rewind_test SELECT FROM generate_series(1, 10); INSERT INTO rewind_test SELECT FROM generate_series(1, 10); INSERT INTO rewind_test SELECT FROM generate_series(1, 10);\"" - ) - dirty_count = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT count(*) FROM rewind_test;\"" - ).strip() - print(f"Rows before rewind: {dirty_count}") - queue_len = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT orioledb_get_rewind_queue_length();\"" - ).strip() - evicted_len = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT orioledb_get_rewind_evicted_length();\"" - ).strip() - print(f"Queue length: {queue_len}, evicted length: {evicted_len}") - - # Phase 3: Rewind — the function crashes postgres, || true absorbs psql error - print(f"Calling orioledb_rewind_to_transaction({xid}, {oxid})...") - server.succeed( - f"psql -U supabase_admin -d postgres -c \"SELECT orioledb_rewind_to_transaction({xid}, {oxid});\" 2>&1 || true" - ) - - # Phase 4: Wait for server restart (systemd Restart=always brings it back) - print("Waiting for server restart...") - time.sleep(5) - server.succeed( - "until psql -U supabase_admin -d postgres -t -A -c 'SELECT 1' 2>/dev/null; do sleep 1; done" - ) - print("Server is back up") - - # Verify - post_hash = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT md5(string_agg(x::text, '.' ORDER BY x)) FROM rewind_test;\"" - ).strip() - post_count = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT count(*) FROM rewind_test;\"" - ).strip() - print(f"Rows: {post_count} (expect 100)") - print(f"Hash before: {pre_hash}") - print(f"Hash after: {post_hash}") - match = "YES" if pre_hash == post_hash else "NO" - print(f"Match: {match}") - assert post_count == "100", f"Expected 100 rows after rewind, got: {post_count}" - assert pre_hash == post_hash, f"Hash mismatch: {pre_hash} != {post_hash}" - print("Basic rewind test PASSED") - - with subtest("Rewind under buffer pressure"): - # Create a new table for buffer pressure test - server.succeed( - "psql -U supabase_admin -d postgres -c \"DROP TABLE IF EXISTS pressure_test; CREATE TABLE pressure_test(x serial) USING orioledb;\"" - ) - - # Insert data in 50 batches of 100 rows each (5000 rows across 50 transactions) - for batch in range(50): - server.succeed( - "psql -U supabase_admin -d postgres -c \"INSERT INTO pressure_test SELECT FROM generate_series(1, 100);\"" - ) - - # Capture checkpoint state in a single query - cp_ids = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT pg_current_xact_id()::text::int4, orioledb_get_current_oxid()::int8, count(*) FROM pressure_test;\"" - ).strip() - cp_parts = cp_ids.split("|") - cp_xid = cp_parts[0] - cp_oxid = cp_parts[1] - cp_count = cp_parts[2] - print(f"Buffer pressure checkpoint: xid={cp_xid}, oxid={cp_oxid}, count={cp_count}") - - # Insert 10 more batches after checkpoint - for batch in range(10): - server.succeed( - "psql -U supabase_admin -d postgres -c \"INSERT INTO pressure_test SELECT FROM generate_series(1, 100);\"" - ) - - after_count = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT count(*) FROM pressure_test;\"" - ).strip() - print(f"After additional inserts: count={after_count}") - - # Print queue/evicted lengths to show buffer state - queue_len = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT orioledb_get_rewind_queue_length();\"" - ).strip() - evicted_len = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT orioledb_get_rewind_evicted_length();\"" - ).strip() - print(f"Buffer pressure state - queue length: {queue_len}, evicted length: {evicted_len}") - - # Rewind — function crashes postgres - print(f"Calling orioledb_rewind_to_transaction({cp_xid}, {cp_oxid})...") - server.succeed( - f"psql -U supabase_admin -d postgres -c \"SELECT orioledb_rewind_to_transaction({cp_xid}, {cp_oxid});\" 2>&1 || true" - ) - - # Wait for server restart - time.sleep(5) - server.succeed( - "until psql -U supabase_admin -d postgres -t -A -c 'SELECT 1' 2>/dev/null; do sleep 1; done" - ) - - # Verify rewind restored checkpoint state - post_pressure_count = server.succeed( - "psql -U supabase_admin -d postgres -t -A -c \"SELECT count(*) FROM pressure_test;\"" - ).strip() - print(f"After buffer pressure rewind: count={post_pressure_count} (expect {cp_count})") - assert post_pressure_count == cp_count, ( - f"Expected {cp_count} rows after buffer pressure rewind, got: {post_pressure_count}" - ) - print("Buffer pressure rewind test PASSED") - ''; -} diff --git a/nix/tests/expected/z_multigres-orioledb-17_docs-indexes.out b/nix/tests/expected/z_multigres-orioledb-17_docs-indexes.out index 5feec82089..8238a51c74 100644 --- a/nix/tests/expected/z_multigres-orioledb-17_docs-indexes.out +++ b/nix/tests/expected/z_multigres-orioledb-17_docs-indexes.out @@ -18,7 +18,6 @@ create index idx_persons_age on persons (age); create index idx_living_persons_age on persons (age) where deceased is false; create index idx_persons_age_desc on persons (age desc nulls last); reindex index concurrently idx_persons_age; -WARNING: REINDEX CONCURRENTLY is not supported for orioledb tables yet, using a plain REINDEX instead reindex table concurrently persons; -WARNING: REINDEX CONCURRENTLY is not supported for orioledb tables yet, using a plain REINDEX instead +NOTICE: skipping reindex of index "public.persons_pkey" concurrently drop table persons cascade; diff --git a/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out b/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out index ea3ac73d4e..e8596ed536 100644 --- a/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out +++ b/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out @@ -1123,6 +1123,7 @@ order by orioledb | extensions | orioledb_tree_stat | relid regclass, OUT level integer, OUT count bigint, OUT avgoccupied double precision, OUT avgvacated double precision | SETOF record orioledb | extensions | orioledb_ucm_check | | boolean orioledb | extensions | orioledb_undo_size | OUT undo_type text, OUT undo_size bigint | SETOF record + orioledb | extensions | orioledb_upgrade_refresh | | void orioledb | extensions | orioledb_version | | text orioledb | extensions | orioledb_write_pages | relid oid | void orioledb | extensions | pg_stopevent_reset | eventname text | boolean @@ -4866,7 +4867,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(4706 rows) +(4707 rows) /* diff --git a/nix/tests/expected/z_orioledb-17_docs-indexes.out b/nix/tests/expected/z_orioledb-17_docs-indexes.out index 5feec82089..8238a51c74 100644 --- a/nix/tests/expected/z_orioledb-17_docs-indexes.out +++ b/nix/tests/expected/z_orioledb-17_docs-indexes.out @@ -18,7 +18,6 @@ create index idx_persons_age on persons (age); create index idx_living_persons_age on persons (age) where deceased is false; create index idx_persons_age_desc on persons (age desc nulls last); reindex index concurrently idx_persons_age; -WARNING: REINDEX CONCURRENTLY is not supported for orioledb tables yet, using a plain REINDEX instead reindex table concurrently persons; -WARNING: REINDEX CONCURRENTLY is not supported for orioledb tables yet, using a plain REINDEX instead +NOTICE: skipping reindex of index "public.persons_pkey" concurrently drop table persons cascade; diff --git a/nix/tests/expected/z_orioledb-17_ext_interface.out b/nix/tests/expected/z_orioledb-17_ext_interface.out index 0482d0b86b..84eeaac1e3 100644 --- a/nix/tests/expected/z_orioledb-17_ext_interface.out +++ b/nix/tests/expected/z_orioledb-17_ext_interface.out @@ -1123,6 +1123,7 @@ order by orioledb | extensions | orioledb_tree_stat | relid regclass, OUT level integer, OUT count bigint, OUT avgoccupied double precision, OUT avgvacated double precision | SETOF record orioledb | extensions | orioledb_ucm_check | | boolean orioledb | extensions | orioledb_undo_size | OUT undo_type text, OUT undo_size bigint | SETOF record + orioledb | extensions | orioledb_upgrade_refresh | | void orioledb | extensions | orioledb_version | | text orioledb | extensions | orioledb_write_pages | relid oid | void orioledb | extensions | pg_stopevent_reset | eventname text | boolean @@ -5003,7 +5004,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(4843 rows) +(4844 rows) /* diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 333f062bc7..6090cab8f0 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -256,6 +256,7 @@ orioledb_config_items() { sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ pgjwt,//g;' "$DATDIR/supautils.conf" sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "$DATDIR/postgresql.conf" echo "default_table_access_method = 'orioledb'" >> "$DATDIR/postgresql.conf" + echo "output_plugin_libraries = 'pgoutput, test_decoding, wal2json'" >> "$DATDIR/postgresql.conf" elif [[ "$1" = "orioledb-17" && "$CURRENT_SYSTEM" = "aarch64-darwin" ]]; then # macOS specific configuration echo "macOS detected, applying macOS specific configuration" @@ -271,6 +272,7 @@ orioledb_config_items() { perl -pi -e 's/(shared_preload_libraries\s*=\s*'\''.*?)'\''/\1, orioledb'\''/' "$DATDIR/postgresql.conf" echo "default_table_access_method = 'orioledb'" >> "$DATDIR/postgresql.conf" + echo "output_plugin_libraries = 'pgoutput, test_decoding, wal2json'" >> "$DATDIR/postgresql.conf" elif [[ "$VERSION" == "17" && "$CURRENT_SYSTEM" != "aarch64-darwin" ]]; then echo "non-macos pg 17 conf" sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf"