Make the vector probe prune, and write decoupled clustered rows in cluster order

ci/journey.sh

@@ -5843,6 +5843,48 @@ story_row_group_pruning() {
     q "$HOST" "SELECT coldfront.drop_iceberg_table('public','tcrg', true);" >/dev/null 2>&1
 }
 
+# ───────────────────────────────────────────────────────────────────────────
+# Story TC-193: the vector probe and row-group skipping. A clustered cold write
+# lands in cluster order, so a row group holds one cluster or two and its
+# statistics on the cluster column say which. The probe's IN predicate reaches
+# the reader, so the row groups of the clusters it does not visit are skipped;
+# the rows with no assignment are a second arm of the cold scan, counted from
+# the same profile, and cost nothing when there are none. Both modes, on its
+# own throwaway table.
+# ───────────────────────────────────────────────────────────────────────────
+story_vector_probe_pruning() {
+    step "TC-193: a vector probe skips the row groups of the clusters it does not visit"
+    local i
+    for i in 1 2 3 4 5; do
+        q_may "$HOST" "SELECT coldfront.create_iceberg_table('public','tcvp','[{\"name\":\"id\",\"type\":\"bigint\"},{\"name\":\"ts\",\"type\":\"timestamptz\"},{\"name\":\"embedding\",\"type\":\"vector(3)\"}]'::jsonb);" >/dev/null 2>&1
+        [ "$(q "$HOST" "SELECT count(*) FROM pg_class WHERE relname='tcvp' AND relkind='v';")" = "1" ] && break
+        sleep 2
+    done
+    # Two clusters far apart; a probe visits one of them.
+    q "$HOST" "INSERT INTO coldfront.vector_config (schema_name, table_name, column_name, nlist, nprobe) VALUES ('public','tcvp','embedding',2,1);" >/dev/null
+    q "$HOST" "INSERT INTO coldfront.vector_centroids (schema_name, table_name, column_name, generation, centroid_id, centroid) VALUES ('public','tcvp','embedding',1,0,ARRAY[100,100,100]::real[]),('public','tcvp','embedding',1,1,ARRAY[-100,-100,-100]::real[]);" >/dev/null
+    q "$HOST" "UPDATE coldfront.vector_config SET generation = 1 WHERE table_name = 'tcvp';" >/dev/null
+    # 300,000 rows alternating between the clusters, in one write: one file with
+    # several row groups, and the write's own ORDER BY is what puts each cluster's
+    # rows together.
+    q "$HOST" "INSERT INTO public.tcvp SELECT i, now(), (CASE WHEN i % 2 = 0 THEN '[100,100,' || (100 + i % 7) || ']' ELSE '[-100,-100,' || (-100 - i % 7) || ']' END)::vector FROM generate_series(1, 300000) i;" >/dev/null 2>&1
+    assert_eq "TC-193: 300,000 rows in one file" "1" "$(ice_files ice.public.tcvp '\.parquet')"
+    # One session: the profile settings, the probe, and the profile it wrote. The
+    # probed read has two ICEBERG_SCAN nodes, the probe arm first and the
+    # unassigned arm second, each read on its own (strict jsonpath: lax .**
+    # yields every array element twice).
+    local counters s1 t1 s2 t2
+    counters=$(q "$HOST" "SELECT duckdb.raw_query('SET custom_profiling_settings = ''{\"OPERATOR_TYPE\": \"true\", \"OPERATOR_ROW_GROUPS_SCANNED\": \"true\", \"OPERATOR_TOTAL_ROW_GROUPS_TO_SCAN\": \"true\"}'''); SELECT duckdb.raw_query('SET enable_profiling = ''json'''); SELECT duckdb.raw_query('SET profiling_output = ''/tmp/tc193.json'''); SELECT id FROM public.tcvp ORDER BY embedding <=> ARRAY[100,100,100]::real[] LIMIT 1; SELECT duckdb.raw_query('SET enable_profiling = ''no_output'''); SELECT string_agg((j->>'operator_row_groups_scanned') || ' ' || (j->>'operator_total_row_groups_to_scan'), ' ' ORDER BY o) FROM jsonb_path_query(pg_read_file('/tmp/tc193.json')::jsonb, 'strict \$.** ? (@.operator_name == \"ICEBERG_SCAN\")') WITH ORDINALITY AS t(j, o);" | tail -1)
+    read -r s1 t1 s2 t2 <<< "$counters"
+    assert_gt "TC-193: the file holds more than one row group" 1 "$t1"
+    assert_gt "TC-193: the probe arm read at least one row group" 0 "$s1"
+    assert_gt "TC-193: the probe arm skipped the row groups of the cluster it did not visit" "$s1" "$t1"
+    assert_eq "TC-193: the unassigned arm scanned nothing" "0" "$t2"
+    q "$HOST" "SELECT coldfront.drop_iceberg_table('public','tcvp', true);" >/dev/null 2>&1
+    assert_eq "TC-193: the drop took the table's vector configuration and centroids with it" "0" \
+        "$(q "$HOST" "SELECT (SELECT count(*) FROM coldfront.vector_config WHERE table_name = 'tcvp') + (SELECT count(*) FROM coldfront.vector_centroids WHERE table_name = 'tcvp');")"
+}
+
 # ── orchestrate ────────────────────────────────────────────────────────────
 # Setup is shared. The story set then branches on mode: tiered exercises the
 # hot+cold partitioned path; decoupled exercises the all-Iceberg wrapper. (The
@@ -5936,6 +5978,7 @@ story_duckdb_temp_dirs     # TC-152: per-backend spill dir; departed backends' s
 story_duckdb_spill_concurrency  # TC-153: four sessions spilling at once stay isolated and correct
 story_partitioned_cold_tables  # TC-186..TC-189: partition fan-out, UTC months, refusals, manifest skipping
 story_row_group_pruning        # TC-192: a time band skips the row groups outside it
+story_vector_probe_pruning     # TC-193: a vector probe skips the row groups of the clusters it does not visit
 story_drop_iceberg_table   # both modes, purge and keep-files (own throwaway tables)
 [ "$MESH" = 1 ] && [ "$MODE" = decoupled ] && story_mesh   # tiered+mesh runs story_mesh_tiered (above)
 [ "$MESH" = 1 ] && story_mesh_multiwriter   # >1 cold writer/node cross-node (tiered: events, decoupled: iceonly)

docs/architecture_decoupled.md

@@ -92,7 +92,7 @@ notes:
 | SELECT (function-call form) | `SELECT … FROM iceberg_scan('ice.<ns>.<name>') r WHERE r['col'] = …` | Columns must use `r['col']` accessor |
 | SELECT (raw-query form) | `SELECT duckdb.raw_query('SELECT ... FROM ice.<ns>.<name> WHERE ...')` | Returns scalar/text result via pg_duckdb's NOTICE channel |
 | ROLLBACK of writes | `BEGIN; raw_query(...); ROLLBACK;` | pg_duckdb's `XactCallback` ties DuckDB↔PG tx, so ROLLBACK undoes pending Iceberg writes |
-| DROP TABLE | `SELECT duckdb.raw_query('DROP TABLE ice.<ns>.<name>')` | |
+| DROP TABLE | `SELECT coldfront.drop_iceberg_table('<schema>', '<name>', <purge>)` | Removes the wrapper view and every registration row, vector configuration included. A raw `DROP TABLE` through `duckdb.raw_query` drops only the catalog table and leaves them behind |
 
 ### What does not work
 
@@ -374,9 +374,9 @@ before the drop is applied there.
 
 ### Handing a table back
 
-`coldfront.release_iceberg_table()` removes the wrapper view and the
-registry row and performs no Iceberg I/O, so the Iceberg table keeps
-every row:
+`coldfront.release_iceberg_table()` removes the wrapper view, the
+registry row and the relation's vector configuration, and performs no
+Iceberg I/O, so the Iceberg table keeps every row:
 
 ```sql
 SELECT coldfront.release_iceberg_table('public', 'orders');

docs/architecture_vectors.md

@@ -262,12 +262,14 @@ Three properties, set at `CREATE TABLE`:
 | `coldfront.sort-key` | the cluster column, then the key | the compactor |
 
 Row groups are the pruning granularity: the Parquet reader skips a row group
-whose statistics cannot match the filter, and at 2048 rows a group holds a median
-of one cluster. The two writers each read one row-group property and ignore the
-other. DuckDB reads only `write.parquet.row-group-size-bytes`, and a table
-carrying that property refuses every DuckDB write to it (`ROW_GROUP_SIZE_BYTES
-does not work while preserving insertion order`), so it is not set. A DuckDB
-write therefore emits one row group per file and compaction is what cuts them.
+whose statistics cannot match the filter. The two writers each read one
+row-group property and ignore the other. iceberg-go honours the 2048-row limit,
+so a compacted file's groups hold a median of one cluster. DuckDB reads only
+`write.parquet.row-group-size-bytes`, and a table carrying that property refuses
+every DuckDB write to it (`ROW_GROUP_SIZE_BYTES does not work while preserving
+insertion order`), so it is not set: a DuckDB write emits its own row groups of
+up to 122,880 rows, each a contiguous slice of the ordered stream, and
+compaction is what cuts them down.
 
 The file target is large because on object storage every file a query touches is
 a billed round trip. A partitioned table (every tiered table, and a decoupled
@@ -282,11 +284,12 @@ scatters a cluster's rows through key space.
 Properties cannot be altered after creation on this build, so a table that
 predates its vector column keeps the defaults.
 
-**Batch cold writes order by cluster.** The archiver's Iceberg INSERT and the C
-bulk INSERT append `ORDER BY 1` (the cluster leads the projection) plus the key,
-so each new file is internally sorted and its own row groups prune. No existing
-file is touched: sorted regions accumulate, and a probe reads the matching row
-group in each of them.
+**Batch cold writes order by cluster.** The archiver's Iceberg INSERT appends
+`ORDER BY 1` (the cluster leads the projection) plus the key, and the C bulk
+INSERT and the decoupled INSERT append `ORDER BY 1`, so each new file is
+internally sorted and its own row groups prune. No existing file is touched:
+sorted regions accumulate, and a probe reads the matching row groups in each of
+them.
 
 **Compaction merges those regions rather than appending them.** A table carrying
 `coldfront.sort-key` is rewritten group by group through `rewriteSorted`
@@ -352,13 +355,16 @@ from a parameter could not have run at all.
 The rewrite resolves the nearest `nprobe` centroid ids
 (`coldfront._vec_probe_ids`), turns them into a predicate
 (`coldfront._vec_probe_qual`), and substitutes the view reference for the view's
-own definition carrying that predicate on its cold arm
+own definition with its cold arm twice: once carrying that predicate, and once
+carrying `IS NULL` on the cluster column for the rows with no assignment
 (`coldfront._vec_probed_viewdef`):
 
 ```sql
 … WHERE r['ts'] < <cutoff>
-  AND (r['_cf_vec_list_embedding']::integer IN (3, 17)
-       OR r['_cf_vec_list_embedding']::integer IS NULL)
+  AND (r['_cf_vec_list_embedding']::integer IN (3, 17))
+UNION ALL
+… WHERE r['ts'] < <cutoff>
+  AND r['_cf_vec_list_embedding']::integer IS NULL
 ```
 
 The substitution exists because the predicate has nowhere else to go: the cluster
@@ -371,10 +377,13 @@ result.
 The hot arm is untouched: hot rows carry no assignment and every one of them is
 returned.
 
-**The null disjunct is not optional.** Rows another engine appended straight to
-Iceberg carry no assignment, and a bare `IN` drops them silently. It is also not
-expensive, because the reader prunes on each row group's null count: unassigned
-rows are read in proportion to their own size rather than the table's.
+**The unassigned arm is not optional, and it is a second arm rather than an
+OR.** Rows another engine appended straight to Iceberg carry no assignment, and
+a bare `IN` drops them silently. As its own arm it is also not expensive: the
+reader prunes it on each file's null count, so unassigned rows are read in
+proportion to their own size, and a table with none reads nothing for it. It is
+not an `OR` on the first arm because DuckDB pushes an `IN` into the scan and to
+the manifest bounds, but not an `OR` that carries `IS NULL`.
 
 **Declining is total and silent.** No centroid generation, an empty probe set, a
 view with no cold arm: each keeps today's query. This is the one place in the

docs/usage.md

@@ -452,9 +452,9 @@ them.
 ### Handing an adopted table back
 
 An adopted table is released rather than dropped, because ColdFront does
-not own it. `coldfront.release_iceberg_table()` removes the wrapper view
-and the registry row and performs no Iceberg I/O, so the table keeps
-every row and stays in the catalog:
+not own it. `coldfront.release_iceberg_table()` removes the wrapper view,
+the registry row and the relation's vector configuration, and performs no
+Iceberg I/O, so the table keeps every row and stays in the catalog:
 
 ```sql
 SELECT coldfront.release_iceberg_table('public', 'orders');

extension/coldfront/coldfront--1.0.sql

@@ -2502,53 +2502,66 @@ BEGIN
 END;
 $$;
 
--- The predicate a probe set becomes on the cold arm, or NULL for an empty set.
---
--- The null arm is not optional. Rows another engine appended straight to Iceberg
--- carry no assignment, and a bare IN drops them silently. It is also not expensive:
--- the reader prunes on each row group's null count, so unassigned rows are read in
--- proportion to their own size rather than the table's.
---
--- Cast on both arms, matching the cutoff qual the view generator already emits. The
--- subscript yields duckdb.unresolved_type, and the cast is what makes this an
+-- The cluster column of p_column as the cold arm reads it. Cast, matching the
+-- cutoff qual the view generator already emits: the subscript yields
+-- duckdb.unresolved_type, and the cast is what makes a comparison on it an
 -- integer comparison the Parquet reader can take.
+CREATE OR REPLACE FUNCTION coldfront._vec_list_ref(p_column text, p_alias text DEFAULT 'r')
+RETURNS text
+LANGUAGE sql IMMUTABLE AS $$
+    SELECT format('%s[%L]::integer', p_alias, coldfront._vec_list_col(p_column));
+$$;
+
+-- The predicate a probe set becomes on the cold arm, or NULL for an empty set:
+-- an IN, which DuckDB pushes into the scan and checks against the manifest
+-- bounds. The rows with no assignment are a second arm of the cold scan
+-- (_vec_probed_viewdef), not an IS NULL here: DuckDB pushes an OR that carries
+-- one to neither place.
 CREATE OR REPLACE FUNCTION coldfront._vec_probe_qual(
     p_column text, p_ids int[], p_alias text DEFAULT 'r')
 RETURNS text
 LANGUAGE sql IMMUTABLE AS $$
-    WITH c(ref) AS (
-        SELECT format('%s[%L]::integer', p_alias, coldfront._vec_list_col(p_column)))
-    SELECT format('(%s IN (%s) OR %s IS NULL)',
-                  c.ref, array_to_string(p_ids, ', '), c.ref)
-      FROM c
+    SELECT format('(%s IN (%s))',
+                  coldfront._vec_list_ref(p_column, p_alias), array_to_string(p_ids, ', '))
      WHERE cardinality(p_ids) > 0;
 $$;
 
--- The view's own definition with a probe predicate on its cold arm, or NULL when
--- there is no cold arm to probe. The read rewrite substitutes this for the view
--- reference, and that substitution is what keeps the cluster column out of the
--- view: the predicate is added where the column already exists, instead of the
--- view exposing a column so a caller's query can name it.
+-- The view's own definition with its cold arm probed, or NULL when there is no
+-- cold arm to probe. The read rewrite substitutes this for the view reference,
+-- and that substitution is what keeps the cluster column out of the view: the
+-- predicate is added where the column already exists, instead of the view
+-- exposing a column so a caller's query can name it.
+--
+-- The cold arm appears twice: once with the probe set, once with IS NULL on the
+-- cluster column for the rows with no assignment. Two arms rather than one OR,
+-- because DuckDB pushes the IN into the scan and to the manifest bounds but not
+-- an OR that carries IS NULL, and the second arm reads only the files whose null
+-- counts say they hold unassigned rows: nothing, when there are none.
 --
 -- Appended, not spliced. The generator puts the cold arm last and gives the view
 -- neither ORDER BY nor LIMIT, so the end of the definition is the end of the cold
 -- arm: of its WHERE for a tiered view, which always carries the cutoff qual, and of
 -- its FROM for a decoupled one, which carries no qual at all. The registry says
--- which, so nothing here parses the deparsed text to find out. The regress test's
--- expected output locks the shape.
+-- which. A tiered view is one set operation, hot arm then cold arm, so its cold
+-- arm is the text after the UNION ALL. The regress test's expected output locks
+-- the shape.
 --
 -- A tiered view with no cutoff has no cold arm at all, only the hot heap, so there
 -- is nothing to probe and the caller keeps its query.
 CREATE OR REPLACE FUNCTION coldfront._vec_probed_viewdef(
-    p_schema text, p_view text, p_qual text)
+    p_schema text, p_view text, p_column text, p_ids int[])
 RETURNS text
 LANGUAGE plpgsql STABLE AS $$
 DECLARE
     v_iceberg_only boolean;
     v_has_cutoff   boolean;
     v_body         text;
+    v_cold         text;
+    v_probed       text;
+    v_unassigned   text;
 BEGIN
-    IF p_qual IS NULL THEN
+    v_probed := coldfront._vec_probe_qual(p_column, p_ids);
+    IF v_probed IS NULL THEN
         RETURN NULL;
     END IF;
 
@@ -2565,9 +2578,14 @@ BEGIN
 
     v_body := rtrim(pg_get_viewdef(format('%I.%I', p_schema, p_view)::regclass),
                     E' \t\r\n;');
-    RETURN v_body
-        || CASE WHEN v_iceberg_only THEN ' WHERE ' ELSE ' AND ' END
-        || p_qual;
+    v_unassigned := coldfront._vec_list_ref(p_column) || ' IS NULL';
+    IF v_iceberg_only THEN
+        RETURN format('%s WHERE %s UNION ALL %s WHERE %s',
+                      v_body, v_probed, v_body, v_unassigned);
+    END IF;
+    v_cold := substring(v_body from '.*\n *UNION ALL\n(.*)$');
+    RETURN format('%s AND %s UNION ALL %s AND %s',
+                  v_body, v_probed, v_cold, v_unassigned);
 END;
 $$;
 
@@ -4112,6 +4130,14 @@ BEGIN
     DELETE FROM coldfront.tiered_views
      WHERE schema_name = p_schema AND relname = p_table;
 
+    -- The registration's vector layout goes with it: the centroids and the
+    -- configuration describe the table this registration named, and a relation
+    -- registered later under the same name must not inherit them.
+    DELETE FROM coldfront.vector_centroids
+     WHERE schema_name = p_schema AND table_name = p_table;
+    DELETE FROM coldfront.vector_config
+     WHERE schema_name = p_schema AND table_name = p_table;
+
     IF NOT v_iceberg_only THEN
         DELETE FROM coldfront.partition_config
          WHERE schema_name = p_schema AND table_name = p_table;