From bff882afae3f645823de635c1b1ff396c6fcf7c3 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 25 Sep 2026 22:47:03 +0100 Subject: [PATCH 1/5] fix: make the vector probe prune, order decoupled clustered writes --- ci/journey.sh | 40 +++++++++ docs/architecture_decoupled.md | 6 +- docs/architecture_vectors.md | 45 ++++++---- docs/usage.md | 6 +- extension/coldfront/coldfront--1.0.sql | 78 ++++++++++------ extension/coldfront/src/coldfront.c | 19 ++-- .../test/expected/drop_iceberg_table.out | 21 ++++- .../coldfront/test/expected/vector_probe.out | 88 +++++++++++-------- .../coldfront/test/sql/drop_iceberg_table.sql | 11 ++- extension/coldfront/test/sql/vector_probe.sql | 28 +++--- 10 files changed, 236 insertions(+), 106 deletions(-) diff --git a/ci/journey.sh b/ci/journey.sh index ad51723..9a19372 100755 --- a/ci/journey.sh +++ b/ci/journey.sh @@ -5843,6 +5843,45 @@ story_row_group_pruning() { q "$HOST" "SELECT coldfront.drop_iceberg_table('public','tcrg', true);" >/dev/null 2>&1 } +# ─────────────────────────────────────────────────────────────────────────── +# Story TC-193: the vector probe and row-group skipping. A clustered cold write +# lands in cluster order, so a row group holds one cluster or two and its +# statistics on the cluster column say which. The probe's IN predicate reaches +# the reader, so the row groups of the clusters it does not visit are skipped; +# the rows with no assignment are a second arm of the cold scan, counted from +# the same profile, and cost nothing when there are none. Both modes, on its +# own throwaway table. +# ─────────────────────────────────────────────────────────────────────────── +story_vector_probe_pruning() { + step "TC-193: a vector probe skips the row groups of the clusters it does not visit" + local i + for i in 1 2 3 4 5; do + q_may "$HOST" "SELECT coldfront.create_iceberg_table('public','tcvp','[{\"name\":\"id\",\"type\":\"bigint\"},{\"name\":\"ts\",\"type\":\"timestamptz\"},{\"name\":\"embedding\",\"type\":\"vector(3)\"}]'::jsonb);" >/dev/null 2>&1 + [ "$(q "$HOST" "SELECT count(*) FROM pg_class WHERE relname='tcvp' AND relkind='v';")" = "1" ] && break + sleep 2 + done + # Two clusters far apart; a probe visits one of them. + q "$HOST" "INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) VALUES ('public','tcvp','embedding',2,1);" >/dev/null + q "$HOST" "INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) VALUES ('public','tcvp','embedding',1,0,ARRAY[100,100,100]::real[]),('public','tcvp','embedding',1,1,ARRAY[-100,-100,-100]::real[]);" >/dev/null + q "$HOST" "UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'tcvp';" >/dev/null + # 300,000 rows alternating between the clusters, in one write: one file with + # several row groups, and the write's own ORDER BY is what puts each cluster's + # rows together. + q "$HOST" "INSERT INTO public.tcvp SELECT i, now(), (CASE WHEN i % 2 = 0 THEN '[100,100,' || (100 + i % 7) || ']' ELSE '[-100,-100,' || (-100 - i % 7) || ']' END)::vector FROM generate_series(1, 300000) i;" >/dev/null 2>&1 + assert_eq "TC-193: 300,000 rows in one file" "1" "$(ice_files ice.public.tcvp '\.parquet')" + # One session: the profile settings, the probe, and the profile it wrote. The + # probed read has two ICEBERG_SCAN nodes; the counters are summed over both. + local counters + counters=$(q "$HOST" "SELECT duckdb.raw_query('SET custom_profiling_settings = ''{\"OPERATOR_TYPE\": \"true\", \"OPERATOR_ROW_GROUPS_SCANNED\": \"true\", \"OPERATOR_TOTAL_ROW_GROUPS_TO_SCAN\": \"true\"}'''); SELECT duckdb.raw_query('SET enable_profiling = ''json'''); SELECT duckdb.raw_query('SET profiling_output = ''/tmp/tc193.json'''); SELECT id FROM public.tcvp ORDER BY embedding <=> ARRAY[100,100,100]::real[] LIMIT 1; SELECT duckdb.raw_query('SET enable_profiling = ''no_output'''); SELECT sum((j->>'operator_row_groups_scanned')::int) || ' ' || sum((j->>'operator_total_row_groups_to_scan')::int) FROM jsonb_path_query(pg_read_file('/tmp/tc193.json')::jsonb, '\$.** ? (@.operator_name == \"ICEBERG_SCAN\")') AS j;" | tail -1) + local scanned=${counters% *} total=${counters#* } + assert_gt "TC-193: the file holds more than one row group" 1 "$total" + assert_gt "TC-193: the probe read at least one row group" 0 "$scanned" + assert_gt "TC-193: the probe skipped the row groups of the cluster it did not visit" "$scanned" "$total" + q "$HOST" "SELECT coldfront.drop_iceberg_table('public','tcvp', true);" >/dev/null 2>&1 + assert_eq "TC-193: the drop took the table's vector configuration and centroids with it" "0" \ + "$(q "$HOST" "SELECT (SELECT count(*) FROM coldfront.vector_config WHERE table_name = 'tcvp') + (SELECT count(*) FROM coldfront.vector_centroids WHERE table_name = 'tcvp');")" +} + # ── orchestrate ──────────────────────────────────────────────────────────── # Setup is shared. The story set then branches on mode: tiered exercises the # hot+cold partitioned path; decoupled exercises the all-Iceberg wrapper. (The @@ -5936,6 +5975,7 @@ story_duckdb_temp_dirs # TC-152: per-backend spill dir; departed backends' s story_duckdb_spill_concurrency # TC-153: four sessions spilling at once stay isolated and correct story_partitioned_cold_tables # TC-186..TC-189: partition fan-out, UTC months, refusals, manifest skipping story_row_group_pruning # TC-192: a time band skips the row groups outside it +story_vector_probe_pruning # TC-193: a vector probe skips the row groups of the clusters it does not visit story_drop_iceberg_table # both modes, purge and keep-files (own throwaway tables) [ "$MESH" = 1 ] && [ "$MODE" = decoupled ] && story_mesh # tiered+mesh runs story_mesh_tiered (above) [ "$MESH" = 1 ] && story_mesh_multiwriter # >1 cold writer/node cross-node (tiered: events, decoupled: iceonly) diff --git a/docs/architecture_decoupled.md b/docs/architecture_decoupled.md index 19117a1..98f0c08 100644 --- a/docs/architecture_decoupled.md +++ b/docs/architecture_decoupled.md @@ -374,9 +374,9 @@ before the drop is applied there. ### Handing a table back -`coldfront.release_iceberg_table()` removes the wrapper view and the -registry row and performs no Iceberg I/O, so the Iceberg table keeps -every row: +`coldfront.release_iceberg_table()` removes the wrapper view, the +registry row and the relation's vector configuration, and performs no +Iceberg I/O, so the Iceberg table keeps every row: ```sql SELECT coldfront.release_iceberg_table('public', 'orders'); diff --git a/docs/architecture_vectors.md b/docs/architecture_vectors.md index 98dbeaa..c002863 100644 --- a/docs/architecture_vectors.md +++ b/docs/architecture_vectors.md @@ -262,12 +262,14 @@ Three properties, set at `CREATE TABLE`: | `coldfront.sort-key` | the cluster column, then the key | the compactor | Row groups are the pruning granularity: the Parquet reader skips a row group -whose statistics cannot match the filter, and at 2048 rows a group holds a median -of one cluster. The two writers each read one row-group property and ignore the -other. DuckDB reads only `write.parquet.row-group-size-bytes`, and a table -carrying that property refuses every DuckDB write to it (`ROW_GROUP_SIZE_BYTES -does not work while preserving insertion order`), so it is not set. A DuckDB -write therefore emits one row group per file and compaction is what cuts them. +whose statistics cannot match the filter. The two writers each read one +row-group property and ignore the other. iceberg-go honours the 2048-row limit, +so a compacted file's groups hold a median of one cluster. DuckDB reads only +`write.parquet.row-group-size-bytes`, and a table carrying that property refuses +every DuckDB write to it (`ROW_GROUP_SIZE_BYTES does not work while preserving +insertion order`), so it is not set: a DuckDB write emits its own row groups of +up to 122,880 rows, each a contiguous slice of the ordered stream, and +compaction is what cuts them down. The file target is large because on object storage every file a query touches is a billed round trip. A partitioned table (every tiered table, and a decoupled @@ -282,11 +284,12 @@ scatters a cluster's rows through key space. Properties cannot be altered after creation on this build, so a table that predates its vector column keeps the defaults. -**Batch cold writes order by cluster.** The archiver's Iceberg INSERT and the C -bulk INSERT append `ORDER BY 1` (the cluster leads the projection) plus the key, -so each new file is internally sorted and its own row groups prune. No existing -file is touched: sorted regions accumulate, and a probe reads the matching row -group in each of them. +**Batch cold writes order by cluster.** The archiver's Iceberg INSERT appends +`ORDER BY 1` (the cluster leads the projection) plus the key, and the C bulk +INSERT and the decoupled INSERT append `ORDER BY 1`, so each new file is +internally sorted and its own row groups prune. No existing file is touched: +sorted regions accumulate, and a probe reads the matching row groups in each of +them. **Compaction merges those regions rather than appending them.** A table carrying `coldfront.sort-key` is rewritten group by group through `rewriteSorted` @@ -352,13 +355,16 @@ from a parameter could not have run at all. The rewrite resolves the nearest `nprobe` centroid ids (`coldfront._vec_probe_ids`), turns them into a predicate (`coldfront._vec_probe_qual`), and substitutes the view reference for the view's -own definition carrying that predicate on its cold arm +own definition with its cold arm twice: once carrying that predicate, and once +carrying `IS NULL` on the cluster column for the rows with no assignment (`coldfront._vec_probed_viewdef`): ```sql … WHERE r['ts'] < - AND (r['_cf_vec_list_embedding']::integer IN (3, 17) - OR r['_cf_vec_list_embedding']::integer IS NULL) + AND (r['_cf_vec_list_embedding']::integer IN (3, 17)) +UNION ALL +… WHERE r['ts'] < + AND r['_cf_vec_list_embedding']::integer IS NULL ``` The substitution exists because the predicate has nowhere else to go: the cluster @@ -371,10 +377,13 @@ result. The hot arm is untouched: hot rows carry no assignment and every one of them is returned. -**The null disjunct is not optional.** Rows another engine appended straight to -Iceberg carry no assignment, and a bare `IN` drops them silently. It is also not -expensive, because the reader prunes on each row group's null count: unassigned -rows are read in proportion to their own size rather than the table's. +**The unassigned arm is not optional, and it is a second arm rather than an +OR.** Rows another engine appended straight to Iceberg carry no assignment, and +a bare `IN` drops them silently. As its own arm it is also not expensive: the +reader prunes it on each file's null count, so unassigned rows are read in +proportion to their own size, and a table with none reads nothing for it. It is +not an `OR` on the first arm because DuckDB pushes an `IN` into the scan and to +the manifest bounds, but not an `OR` that carries `IS NULL`. **Declining is total and silent.** No centroid generation, an empty probe set, a view with no cold arm: each keeps today's query. This is the one place in the diff --git a/docs/usage.md b/docs/usage.md index 94b1efa..8cf8328 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -452,9 +452,9 @@ them. ### Handing an adopted table back An adopted table is released rather than dropped, because ColdFront does -not own it. `coldfront.release_iceberg_table()` removes the wrapper view -and the registry row and performs no Iceberg I/O, so the table keeps -every row and stays in the catalog: +not own it. `coldfront.release_iceberg_table()` removes the wrapper view, +the registry row and the relation's vector configuration, and performs no +Iceberg I/O, so the table keeps every row and stays in the catalog: ```sql SELECT coldfront.release_iceberg_table('public', 'orders'); diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index f162cee..e48d350 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -2502,53 +2502,66 @@ BEGIN END; $$; --- The predicate a probe set becomes on the cold arm, or NULL for an empty set. --- --- The null arm is not optional. Rows another engine appended straight to Iceberg --- carry no assignment, and a bare IN drops them silently. It is also not expensive: --- the reader prunes on each row group's null count, so unassigned rows are read in --- proportion to their own size rather than the table's. --- --- Cast on both arms, matching the cutoff qual the view generator already emits. The --- subscript yields duckdb.unresolved_type, and the cast is what makes this an +-- The cluster column of p_column as the cold arm reads it. Cast, matching the +-- cutoff qual the view generator already emits: the subscript yields +-- duckdb.unresolved_type, and the cast is what makes a comparison on it an -- integer comparison the Parquet reader can take. +CREATE OR REPLACE FUNCTION coldfront._vec_list_ref(p_column text, p_alias text DEFAULT 'r') +RETURNS text +LANGUAGE sql IMMUTABLE AS $$ + SELECT format('%s[%L]::integer', p_alias, coldfront._vec_list_col(p_column)); +$$; + +-- The predicate a probe set becomes on the cold arm, or NULL for an empty set: +-- an IN, which DuckDB pushes into the scan and checks against the manifest +-- bounds. The rows with no assignment are a second arm of the cold scan +-- (_vec_probed_viewdef), not an IS NULL here: DuckDB pushes an OR that carries +-- one to neither place. CREATE OR REPLACE FUNCTION coldfront._vec_probe_qual( p_column text, p_ids int[], p_alias text DEFAULT 'r') RETURNS text LANGUAGE sql IMMUTABLE AS $$ - WITH c(ref) AS ( - SELECT format('%s[%L]::integer', p_alias, coldfront._vec_list_col(p_column))) - SELECT format('(%s IN (%s) OR %s IS NULL)', - c.ref, array_to_string(p_ids, ', '), c.ref) - FROM c + SELECT format('(%s IN (%s))', + coldfront._vec_list_ref(p_column, p_alias), array_to_string(p_ids, ', ')) WHERE cardinality(p_ids) > 0; $$; --- The view's own definition with a probe predicate on its cold arm, or NULL when --- there is no cold arm to probe. The read rewrite substitutes this for the view --- reference, and that substitution is what keeps the cluster column out of the --- view: the predicate is added where the column already exists, instead of the --- view exposing a column so a caller's query can name it. +-- The view's own definition with its cold arm probed, or NULL when there is no +-- cold arm to probe. The read rewrite substitutes this for the view reference, +-- and that substitution is what keeps the cluster column out of the view: the +-- predicate is added where the column already exists, instead of the view +-- exposing a column so a caller's query can name it. +-- +-- The cold arm appears twice: once with the probe set, once with IS NULL on the +-- cluster column for the rows with no assignment. Two arms rather than one OR, +-- because DuckDB pushes the IN into the scan and to the manifest bounds but not +-- an OR that carries IS NULL, and the second arm reads only the files whose null +-- counts say they hold unassigned rows: nothing, when there are none. -- -- Appended, not spliced. The generator puts the cold arm last and gives the view -- neither ORDER BY nor LIMIT, so the end of the definition is the end of the cold -- arm: of its WHERE for a tiered view, which always carries the cutoff qual, and of -- its FROM for a decoupled one, which carries no qual at all. The registry says --- which, so nothing here parses the deparsed text to find out. The regress test's --- expected output locks the shape. +-- which. A tiered view is one set operation, hot arm then cold arm, so its cold +-- arm is the text after the UNION ALL. The regress test's expected output locks +-- the shape. -- -- A tiered view with no cutoff has no cold arm at all, only the hot heap, so there -- is nothing to probe and the caller keeps its query. CREATE OR REPLACE FUNCTION coldfront._vec_probed_viewdef( - p_schema text, p_view text, p_qual text) + p_schema text, p_view text, p_column text, p_ids int[]) RETURNS text LANGUAGE plpgsql STABLE AS $$ DECLARE v_iceberg_only boolean; v_has_cutoff boolean; v_body text; + v_cold text; + v_probed text; + v_unassigned text; BEGIN - IF p_qual IS NULL THEN + v_probed := coldfront._vec_probe_qual(p_column, p_ids); + IF v_probed IS NULL THEN RETURN NULL; END IF; @@ -2565,9 +2578,14 @@ BEGIN v_body := rtrim(pg_get_viewdef(format('%I.%I', p_schema, p_view)::regclass), E' \t\r\n;'); - RETURN v_body - || CASE WHEN v_iceberg_only THEN ' WHERE ' ELSE ' AND ' END - || p_qual; + v_unassigned := coldfront._vec_list_ref(p_column) || ' IS NULL'; + IF v_iceberg_only THEN + RETURN format('%s WHERE %s UNION ALL %s WHERE %s', + v_body, v_probed, v_body, v_unassigned); + END IF; + v_cold := substring(v_body from '.*\n *UNION ALL\n(.*)$'); + RETURN format('%s AND %s UNION ALL %s AND %s', + v_body, v_probed, v_cold, v_unassigned); END; $$; @@ -4112,6 +4130,14 @@ BEGIN DELETE FROM coldfront.tiered_views WHERE schema_name = p_schema AND relname = p_table; + -- The registration's vector layout goes with it: the centroids and the + -- configuration describe the table this registration named, and a relation + -- registered later under the same name must not inherit them. + DELETE FROM coldfront.vector_centroids + WHERE schema_name = p_schema AND table_name = p_table; + DELETE FROM coldfront.vector_config + WHERE schema_name = p_schema AND table_name = p_table; + IF NOT v_iceberg_only THEN DELETE FROM coldfront.partition_config WHERE schema_name = p_schema AND table_name = p_table; diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 9375ff4..87af2b3 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -2323,9 +2323,12 @@ build_iceberg_only_insert_with_cluster(Query *query, TieredViewInfo *info, if (prefix == NULL || list_cols == NULL) return NULL; + /* Ordered by cluster, so this write's own row groups each hold about one + * cluster and a probe skips the rest of the file. The cluster leads the + * projection, hence ordinal 1. */ initStringInfo(&sql); appendStringInfo(&sql, - "INSERT INTO %s (%s, %s) SELECT %s%s FROM (%s) AS coldfront_src(%s)", + "INSERT INTO %s (%s, %s) SELECT %s%s FROM (%s) AS coldfront_src(%s) ORDER BY 1", info->iceberg_table, list_cols, col_list, prefix, col_list, source, col_list); return sql.data; @@ -3015,9 +3018,12 @@ cf_probe_match(Query *query, char **vec_name, char **vec_lit) * * The predicate cannot be added to the caller's query, because the column it * tests is deliberately in no branch of the view (see coldfront._vec_list_col). - * So the view reference is replaced by the view's own definition carrying the - * predicate on its cold arm, which puts the test where the column exists and - * leaves the caller's query surface alone. Nothing here is text surgery on the + * So the view reference is replaced by the view's own definition with its cold + * arm twice: once carrying the predicate, once carrying IS NULL for the rows + * with no assignment. DuckDB pushes an IN into the scan but not an OR that + * carries IS NULL, and the second arm costs nothing when every row is assigned. + * That puts the test where the column exists and leaves the caller's query + * surface alone. Nothing here is text surgery on the * caller's SQL: the substitution swaps one range-table entry for a subquery and * PostgreSQL deparses the result. * @@ -3049,9 +3055,8 @@ cf_maybe_inject_probe(Query *query) /* Resolve the probe set and the definition that carries it, in one round trip. */ initStringInfo(&q); appendStringInfo(&q, - "SELECT coldfront._vec_probed_viewdef(%s, %s, " - "coldfront._vec_probe_qual(%s, coldfront._vec_probe_ids(" - "%s, %s, %s, %s::real[], %s)))", + "SELECT coldfront._vec_probed_viewdef(%s, %s, %s, " + "coldfront._vec_probe_ids(%s, %s, %s, %s::real[], %s))", quote_literal_cstr(get_namespace_name( get_rel_namespace(view_rte->relid))), quote_literal_cstr(get_rel_name(view_rte->relid)), diff --git a/extension/coldfront/test/expected/drop_iceberg_table.out b/extension/coldfront/test/expected/drop_iceberg_table.out index 876487e..a2f8527 100644 --- a/extension/coldfront/test/expected/drop_iceberg_table.out +++ b/extension/coldfront/test/expected/drop_iceberg_table.out @@ -76,7 +76,14 @@ VALUES ('public', 'iceonly_again', NULL, '"ice"."public"."iceonly"', NULL, true) ERROR: duplicate key value violates unique constraint "tiered_views_iceberg_table_key" DETAIL: Key (iceberg_table)=("ice"."public"."iceonly") already exists. DROP VIEW public.iceonly_again; --- Decoupled teardown: the registry row and the wrapper view both go. +-- Decoupled teardown: the registry row and the wrapper view both go, and so do +-- the relation's vector configuration and centroids: they describe the table this +-- registration named, and a relation registered later under the same name must +-- not inherit them. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'iceonly', 'embedding', 2, 1, 1); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'iceonly', 'embedding', 1, 0, ARRAY[1,0,0]::real[]); SELECT coldfront._unregister_iceberg('public', 'iceonly'); _unregister_iceberg --------------------- @@ -95,6 +102,18 @@ SELECT count(*) AS view_left FROM pg_class WHERE relname = 'iceonly'; 0 (1 row) +SELECT count(*) AS vector_config_rows FROM coldfront.vector_config WHERE table_name = 'iceonly'; + vector_config_rows +-------------------- + 0 +(1 row) + +SELECT count(*) AS centroid_rows FROM coldfront.vector_centroids WHERE table_name = 'iceonly'; + centroid_rows +--------------- + 0 +(1 row) + -- Tiered teardown: the archiver's first run renamed events to _events and put a -- view in its place, so un-tiering reverses that. Every registration row goes -- (partition_config too, or the next archiver run re-tiers into a dropped diff --git a/extension/coldfront/test/expected/vector_probe.out b/extension/coldfront/test/expected/vector_probe.out index e0690d8..f9908f9 100644 --- a/extension/coldfront/test/expected/vector_probe.out +++ b/extension/coldfront/test/expected/vector_probe.out @@ -68,19 +68,21 @@ SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[1,0,0]::r (1 row) UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'chunks'; --- The predicate. The null arm is not optional: a row another engine appended --- straight to Iceberg carries no assignment, and a bare IN would drop it. +-- The predicate on the probed arm: an IN the scan takes as a filter. The rows with +-- no assignment are read by a second arm of the cold scan (below), not by an OR: +-- DuckDB pushes an OR that carries IS NULL neither into the scan nor to the +-- manifest bounds. SELECT coldfront._vec_probe_qual('embedding', ARRAY[1,2]) AS qual; - qual --------------------------------------------------------------------------------------------------- - (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) + qual +-------------------------------------------------- + (r['_cf_vec_list_embedding']::integer IN (1, 2)) (1 row) SELECT coldfront._vec_probe_qual('embedding', coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[])) AS qual_from_probe; - qual_from_probe --------------------------------------------------------------------------------------------------- - (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) + qual_from_probe +-------------------------------------------------- + (r['_cf_vec_list_embedding']::integer IN (1, 2)) (1 row) SELECT coldfront._vec_probe_qual('embedding', NULL) IS NULL AS no_probe_set, @@ -114,24 +116,38 @@ SELECT count(*) AS cluster_column_in_view 0 (1 row) --- The probed definition. The tail is what matters: the qual lands inside the cold --- arm's WHERE, after the cutoff comparison, and the hot arm is untouched. -SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])); - _vec_probed_viewdef -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - SELECT _chunks.id, + - _chunks.ts, + - (_chunks.body)::character varying AS body, + - _chunks._cf_vec_embedding AS embedding + - FROM _chunks + - WHERE (_chunks.ts >= '2026-03-01 00:00:00+00'::timestamp with time zone) + - UNION ALL + - SELECT (r.r['id'::text])::bigint AS id, + - (r.r['ts'::text])::timestamp with time zone AS ts, + - (r.r['body'::text])::character varying AS body, + - (r.r['embedding'::text])::real[] AS embedding + - FROM iceberg_scan('ice.default.chunks'::text) r(r) + - WHERE (r.r['ts'::text] < '2026-03-01 00:00:00+00'::timestamp with time zone) AND (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) +-- The probed definition. The cold arm appears twice, once with the probe set and +-- once for the rows with no assignment, each inside its own WHERE after the cutoff +-- comparison; the hot arm appears once and is untouched. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]); + _vec_probed_viewdef +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + SELECT _chunks.id, + + _chunks.ts, + + (_chunks.body)::character varying AS body, + + _chunks._cf_vec_embedding AS embedding + + FROM _chunks + + WHERE (_chunks.ts >= '2026-03-01 00:00:00+00'::timestamp with time zone) + + UNION ALL + + SELECT (r.r['id'::text])::bigint AS id, + + (r.r['ts'::text])::timestamp with time zone AS ts, + + (r.r['body'::text])::character varying AS body, + + (r.r['embedding'::text])::real[] AS embedding + + FROM iceberg_scan('ice.default.chunks'::text) r(r) + + WHERE (r.r['ts'::text] < '2026-03-01 00:00:00+00'::timestamp with time zone) AND (r['_cf_vec_list_embedding']::integer IN (1, 2)) UNION ALL SELECT (r.r['id'::text])::bigint AS id,+ + (r.r['ts'::text])::timestamp with time zone AS ts, + + (r.r['body'::text])::character varying AS body, + + (r.r['embedding'::text])::real[] AS embedding + + FROM iceberg_scan('ice.default.chunks'::text) r(r) + + WHERE (r.r['ts'::text] < '2026-03-01 00:00:00+00'::timestamp with time zone) AND r['_cf_vec_list_embedding']::integer IS NULL +(1 row) + +SELECT (length(d) - length(replace(d, 'FROM _chunks', ''))) / length('FROM _chunks') AS hot_arms, + (length(d) - length(replace(d, 'iceberg_scan(', ''))) / length('iceberg_scan(') AS cold_arms + FROM coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]) AS d; + hot_arms | cold_arms +----------+----------- + 1 | 2 (1 row) -- It reparses, which is the whole contract: the rewrite substitutes this for the @@ -139,25 +155,25 @@ SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qu DO $do$ BEGIN EXECUTE format('CREATE VIEW public.chunks_probed AS %s', - coldfront._vec_probed_viewdef('public', 'chunks', - coldfront._vec_probe_qual('embedding', ARRAY[1,2]))); + coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2])); END $do$; SELECT right(pg_get_viewdef('public.chunks_probed'::regclass), 120) AS reparsed_tail; reparsed_tail -------------------------------------------------------------------------------------------------------------------------- - vec_list_embedding'::text])::integer = ANY (ARRAY[1, 2])) OR ((r.r['_cf_vec_list_embedding'::text])::integer IS NULL))); + xt] < '2026-03-01 00:00:00+00'::timestamp with time zone) AND ((r.r['_cf_vec_list_embedding'::text])::integer IS NULL)); (1 row) --- Declining is silent. No probe set, and a table with no vector column. -SELECT coldfront._vec_probed_viewdef('public', 'chunks', NULL) IS NULL AS no_qual; - no_qual ---------- - t +-- Declining is silent. No probe set, an empty one, and a table with no vector column. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', NULL) IS NULL AS no_probe_set, + coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', '{}') IS NULL AS empty_probe_set; + no_probe_set | empty_probe_set +--------------+----------------- + t | t (1 row) UPDATE coldfront.tiered_views SET vec_columns = NULL WHERE relname = 'chunks'; -SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS no_vector_column; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]) IS NULL AS no_vector_column; no_vector_column ------------------ t @@ -167,7 +183,7 @@ UPDATE coldfront.tiered_views SET vec_columns = ARRAY['embedding'] WHERE relname -- A tiered view with no cutoff is hot-only: it has no cold arm, so there is nothing -- to probe. DELETE FROM coldfront.archive_watermark WHERE table_name = 'chunks'; -SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS hot_only; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]) IS NULL AS hot_only; hot_only ---------- t diff --git a/extension/coldfront/test/sql/drop_iceberg_table.sql b/extension/coldfront/test/sql/drop_iceberg_table.sql index c977fd4..3279e07 100644 --- a/extension/coldfront/test/sql/drop_iceberg_table.sql +++ b/extension/coldfront/test/sql/drop_iceberg_table.sql @@ -59,10 +59,19 @@ INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_tabl VALUES ('public', 'iceonly_again', NULL, '"ice"."public"."iceonly"', NULL, true); DROP VIEW public.iceonly_again; --- Decoupled teardown: the registry row and the wrapper view both go. +-- Decoupled teardown: the registry row and the wrapper view both go, and so do +-- the relation's vector configuration and centroids: they describe the table this +-- registration named, and a relation registered later under the same name must +-- not inherit them. +INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'iceonly', 'embedding', 2, 1, 1); +INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) +VALUES ('public', 'iceonly', 'embedding', 1, 0, ARRAY[1,0,0]::real[]); SELECT coldfront._unregister_iceberg('public', 'iceonly'); SELECT count(*) AS registry_rows FROM coldfront.tiered_views WHERE relname = 'iceonly'; SELECT count(*) AS view_left FROM pg_class WHERE relname = 'iceonly'; +SELECT count(*) AS vector_config_rows FROM coldfront.vector_config WHERE table_name = 'iceonly'; +SELECT count(*) AS centroid_rows FROM coldfront.vector_centroids WHERE table_name = 'iceonly'; -- Tiered teardown: the archiver's first run renamed events to _events and put a -- view in its place, so un-tiering reverses that. Every registration row goes diff --git a/extension/coldfront/test/sql/vector_probe.sql b/extension/coldfront/test/sql/vector_probe.sql index 16a7446..ca787d9 100644 --- a/extension/coldfront/test/sql/vector_probe.sql +++ b/extension/coldfront/test/sql/vector_probe.sql @@ -43,8 +43,10 @@ UPDATE coldfront.vector_config SET generation = 0 WHERE table_name = 'chunks'; SELECT coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[1,0,0]::real[]) IS NULL AS untrained; UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'chunks'; --- The predicate. The null arm is not optional: a row another engine appended --- straight to Iceberg carries no assignment, and a bare IN would drop it. +-- The predicate on the probed arm: an IN the scan takes as a filter. The rows with +-- no assignment are read by a second arm of the cold scan (below), not by an OR: +-- DuckDB pushes an OR that carries IS NULL neither into the scan nor to the +-- manifest bounds. SELECT coldfront._vec_probe_qual('embedding', ARRAY[1,2]) AS qual; SELECT coldfront._vec_probe_qual('embedding', coldfront._vec_probe_ids('public', 'chunks', 'embedding', ARRAY[0.1,0.9,0.2]::real[])) AS qual_from_probe; @@ -67,31 +69,35 @@ SELECT count(*) AS cluster_column_in_view FROM pg_attribute WHERE attrelid = 'public.chunks'::regclass AND attname = coldfront._vec_list_col('embedding'); --- The probed definition. The tail is what matters: the qual lands inside the cold --- arm's WHERE, after the cutoff comparison, and the hot arm is untouched. -SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])); +-- The probed definition. The cold arm appears twice, once with the probe set and +-- once for the rows with no assignment, each inside its own WHERE after the cutoff +-- comparison; the hot arm appears once and is untouched. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]); +SELECT (length(d) - length(replace(d, 'FROM _chunks', ''))) / length('FROM _chunks') AS hot_arms, + (length(d) - length(replace(d, 'iceberg_scan(', ''))) / length('iceberg_scan(') AS cold_arms + FROM coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]) AS d; -- It reparses, which is the whole contract: the rewrite substitutes this for the -- view reference and PostgreSQL has to accept it. IN becomes = ANY on the way in. DO $do$ BEGIN EXECUTE format('CREATE VIEW public.chunks_probed AS %s', - coldfront._vec_probed_viewdef('public', 'chunks', - coldfront._vec_probe_qual('embedding', ARRAY[1,2]))); + coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2])); END $do$; SELECT right(pg_get_viewdef('public.chunks_probed'::regclass), 120) AS reparsed_tail; --- Declining is silent. No probe set, and a table with no vector column. -SELECT coldfront._vec_probed_viewdef('public', 'chunks', NULL) IS NULL AS no_qual; +-- Declining is silent. No probe set, an empty one, and a table with no vector column. +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', NULL) IS NULL AS no_probe_set, + coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', '{}') IS NULL AS empty_probe_set; UPDATE coldfront.tiered_views SET vec_columns = NULL WHERE relname = 'chunks'; -SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS no_vector_column; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]) IS NULL AS no_vector_column; UPDATE coldfront.tiered_views SET vec_columns = ARRAY['embedding'] WHERE relname = 'chunks'; -- A tiered view with no cutoff is hot-only: it has no cold arm, so there is nothing -- to probe. DELETE FROM coldfront.archive_watermark WHERE table_name = 'chunks'; -SELECT coldfront._vec_probed_viewdef('public', 'chunks', coldfront._vec_probe_qual('embedding', ARRAY[1,2])) IS NULL AS hot_only; +SELECT coldfront._vec_probed_viewdef('public', 'chunks', 'embedding', ARRAY[1,2]) IS NULL AS hot_only; -- Cleanup. Unregister before dropping: the DDL hook blocks DROP of a registered -- tiered table/view. From f85a021043bfbb8fb2831b1afae70f52af0a3eee Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 25 Sep 2026 22:59:50 +0100 Subject: [PATCH 2/5] test: expect the IN-only probe qual in vector_multicolumn --- extension/coldfront/test/expected/vector_multicolumn.out | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extension/coldfront/test/expected/vector_multicolumn.out b/extension/coldfront/test/expected/vector_multicolumn.out index 83b96a1..d759d0c 100644 --- a/extension/coldfront/test/expected/vector_multicolumn.out +++ b/extension/coldfront/test/expected/vector_multicolumn.out @@ -72,9 +72,9 @@ CONTEXT: PL/pgSQL function _vec_list_prefix(text,text,text[],text[]) line 10 at -- filter on different cluster columns. SELECT coldfront._vec_probe_qual('embedding', ARRAY[1,2]) AS embedding_qual, coldfront._vec_probe_qual('summary', ARRAY[7]) AS summary_qual; - embedding_qual | summary_qual ---------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------- - (r['_cf_vec_list_embedding']::integer IN (1, 2) OR r['_cf_vec_list_embedding']::integer IS NULL) | (r['_cf_vec_list_summary']::integer IN (7) OR r['_cf_vec_list_summary']::integer IS NULL) + embedding_qual | summary_qual +--------------------------------------------------+--------------------------------------------- + (r['_cf_vec_list_embedding']::integer IN (1, 2)) | (r['_cf_vec_list_summary']::integer IN (7)) (1 row) -- Each column carries its own configuration and its own generation. From dfe3d53df8643ad9a33c9c1ca698746778698ee7 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 25 Sep 2026 22:59:54 +0100 Subject: [PATCH 3/5] test: assert each arm of the vector probe on its own --- ci/journey.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ci/journey.sh b/ci/journey.sh index 9a19372..7099e34 100755 --- a/ci/journey.sh +++ b/ci/journey.sh @@ -5870,13 +5870,16 @@ story_vector_probe_pruning() { q "$HOST" "INSERT INTO public.tcvp SELECT i, now(), (CASE WHEN i % 2 = 0 THEN '[100,100,' || (100 + i % 7) || ']' ELSE '[-100,-100,' || (-100 - i % 7) || ']' END)::vector FROM generate_series(1, 300000) i;" >/dev/null 2>&1 assert_eq "TC-193: 300,000 rows in one file" "1" "$(ice_files ice.public.tcvp '\.parquet')" # One session: the profile settings, the probe, and the profile it wrote. The - # probed read has two ICEBERG_SCAN nodes; the counters are summed over both. - local counters - counters=$(q "$HOST" "SELECT duckdb.raw_query('SET custom_profiling_settings = ''{\"OPERATOR_TYPE\": \"true\", \"OPERATOR_ROW_GROUPS_SCANNED\": \"true\", \"OPERATOR_TOTAL_ROW_GROUPS_TO_SCAN\": \"true\"}'''); SELECT duckdb.raw_query('SET enable_profiling = ''json'''); SELECT duckdb.raw_query('SET profiling_output = ''/tmp/tc193.json'''); SELECT id FROM public.tcvp ORDER BY embedding <=> ARRAY[100,100,100]::real[] LIMIT 1; SELECT duckdb.raw_query('SET enable_profiling = ''no_output'''); SELECT sum((j->>'operator_row_groups_scanned')::int) || ' ' || sum((j->>'operator_total_row_groups_to_scan')::int) FROM jsonb_path_query(pg_read_file('/tmp/tc193.json')::jsonb, '\$.** ? (@.operator_name == \"ICEBERG_SCAN\")') AS j;" | tail -1) - local scanned=${counters% *} total=${counters#* } - assert_gt "TC-193: the file holds more than one row group" 1 "$total" - assert_gt "TC-193: the probe read at least one row group" 0 "$scanned" - assert_gt "TC-193: the probe skipped the row groups of the cluster it did not visit" "$scanned" "$total" + # probed read has two ICEBERG_SCAN nodes, the probe arm first and the + # unassigned arm second, each read on its own (strict jsonpath: lax .** + # yields every array element twice). + local counters s1 t1 s2 t2 + counters=$(q "$HOST" "SELECT duckdb.raw_query('SET custom_profiling_settings = ''{\"OPERATOR_TYPE\": \"true\", \"OPERATOR_ROW_GROUPS_SCANNED\": \"true\", \"OPERATOR_TOTAL_ROW_GROUPS_TO_SCAN\": \"true\"}'''); SELECT duckdb.raw_query('SET enable_profiling = ''json'''); SELECT duckdb.raw_query('SET profiling_output = ''/tmp/tc193.json'''); SELECT id FROM public.tcvp ORDER BY embedding <=> ARRAY[100,100,100]::real[] LIMIT 1; SELECT duckdb.raw_query('SET enable_profiling = ''no_output'''); SELECT string_agg((j->>'operator_row_groups_scanned') || ' ' || (j->>'operator_total_row_groups_to_scan'), ' ' ORDER BY o) FROM jsonb_path_query(pg_read_file('/tmp/tc193.json')::jsonb, 'strict \$.** ? (@.operator_name == \"ICEBERG_SCAN\")') WITH ORDINALITY AS t(j, o);" | tail -1) + read -r s1 t1 s2 t2 <<< "$counters" + assert_gt "TC-193: the file holds more than one row group" 1 "$t1" + assert_gt "TC-193: the probe arm read at least one row group" 0 "$s1" + assert_gt "TC-193: the probe arm skipped the row groups of the cluster it did not visit" "$s1" "$t1" + assert_eq "TC-193: the unassigned arm scanned nothing" "0" "$t2" q "$HOST" "SELECT coldfront.drop_iceberg_table('public','tcvp', true);" >/dev/null 2>&1 assert_eq "TC-193: the drop took the table's vector configuration and centroids with it" "0" \ "$(q "$HOST" "SELECT (SELECT count(*) FROM coldfront.vector_config WHERE table_name = 'tcvp') + (SELECT count(*) FROM coldfront.vector_centroids WHERE table_name = 'tcvp');")" From 81cd710acf686e276dd2cd605a8def6545202df9 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 25 Sep 2026 23:11:07 +0100 Subject: [PATCH 4/5] test: expect the ordered decoupled INSERT in cte_on_insert --- extension/coldfront/test/expected/cte_on_insert.out | 6 +++--- extension/coldfront/test/expected/cte_on_insert_1.out | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extension/coldfront/test/expected/cte_on_insert.out b/extension/coldfront/test/expected/cte_on_insert.out index 941eab8..8052e82 100644 --- a/extension/coldfront/test/expected/cte_on_insert.out +++ b/extension/coldfront/test/expected/cte_on_insert.out @@ -62,10 +62,10 @@ VALUES ('public', 'icevec', 'ice.default.icevec', true, ARRAY['embedding']); EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, ARRAY[1,0,0]::real[] AS embedding) INSERT INTO public.icevec SELECT id, ts, embedding FROM s; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = ''public'' AND vc.table_name = ''icevec'' AND vc.column_name = ''embedding'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding)'::text) + Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = ''public'' AND vc.table_name = ''icevec'' AND vc.column_name = ''embedding'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding) ORDER BY 1'::text) (2 rows) -- Cleanup. diff --git a/extension/coldfront/test/expected/cte_on_insert_1.out b/extension/coldfront/test/expected/cte_on_insert_1.out index e369933..097be38 100644 --- a/extension/coldfront/test/expected/cte_on_insert_1.out +++ b/extension/coldfront/test/expected/cte_on_insert_1.out @@ -65,7 +65,7 @@ EXPLAIN (COSTS OFF, VERBOSE) QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = ''public'' AND vc.table_name = ''icevec'' AND vc.column_name = ''embedding'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding)'::text) + Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = ''public'' AND vc.table_name = ''icevec'' AND vc.column_name = ''embedding'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding) ORDER BY 1'::text) (2 rows) -- Cleanup. From 4f648251c9890bedf216611cf488ede1b3383f67 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 25 Sep 2026 23:11:07 +0100 Subject: [PATCH 5/5] docs: name drop_iceberg_table as the decoupled DROP TABLE path --- docs/architecture_decoupled.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture_decoupled.md b/docs/architecture_decoupled.md index 98f0c08..f2ea522 100644 --- a/docs/architecture_decoupled.md +++ b/docs/architecture_decoupled.md @@ -92,7 +92,7 @@ notes: | SELECT (function-call form) | `SELECT … FROM iceberg_scan('ice..') r WHERE r['col'] = …` | Columns must use `r['col']` accessor | | SELECT (raw-query form) | `SELECT duckdb.raw_query('SELECT ... FROM ice.. WHERE ...')` | Returns scalar/text result via pg_duckdb's NOTICE channel | | ROLLBACK of writes | `BEGIN; raw_query(...); ROLLBACK;` | pg_duckdb's `XactCallback` ties DuckDB↔PG tx, so ROLLBACK undoes pending Iceberg writes | -| DROP TABLE | `SELECT duckdb.raw_query('DROP TABLE ice..')` | | +| DROP TABLE | `SELECT coldfront.drop_iceberg_table('', '', )` | Removes the wrapper view and every registration row, vector configuration included. A raw `DROP TABLE` through `duckdb.raw_query` drops only the catalog table and leaves them behind | ### What does not work