diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58edbe848..e10de01c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,15 @@ jobs: - name: Replay rate migrations on PostgreSQL 17 (Docker) run: node scripts/test-migration-replay.mjs + - name: Replay the DOS-Me Item migration proposal (Docker) + run: node scripts/test-item-migration-proposal.mjs + + - name: Replay the DOS-Me M0 seed re-key proposal (Docker) + run: node scripts/test-m0-seed-rekey-proposal.mjs + + - name: Replay the DOS-Me Item RPC and M3/M4 proposals (Docker) + run: node scripts/test-m3-m4-item-rpcs-proposal.mjs + - name: Check generated rate schema types match the migrations run: pnpm check:db-types @@ -147,3 +156,43 @@ jobs: playwright-report/ test-results/e2e/ retention-days: 7 + + # RATE_ITEM_MODEL=nullable_variant on the DOS-Me M0 + M1+M2 proposals (DOS/Rate#1159): + # the Item routes, redirects and canonicals. The default E2E job above stays flag-off. + e2e-item-model: + name: E2E (Playwright, Item model) + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 10.34.5 + + - name: Setup Node.js environment + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install Playwright Chromium + run: pnpm exec playwright install --with-deps chromium + + - name: Run Item-model browser E2E against a throwaway database + run: node scripts/e2e/run.mjs --item-model + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: playwright-report-item-model + path: | + playwright-report-item-model/ + test-results/e2e-item-model/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index a81e6db7f..48f5c2c3e 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,7 @@ Rate.code-workspace # Playwright HTML report playwright-report/ +playwright-report-item-model/ # Secrets and local data that must never be committed (DOS/Rate#153): # Windows Explorer copies (English "x - Copy.env", Vietnamese "x - Bản sao"), @@ -135,6 +136,19 @@ playwright-report/ !supabase/migrations/*.sql # Reviewed E2E fixture: synthetic catalog rows only, no credentials. !apps/web/e2e/seed.sql +!apps/web/e2e/item-model/seed.sql +# Reviewed DOS-Me migration proposal (DOS/Rate#1150): DDL only, no data, no credentials. +!Docs/Proposals/2026-10-28-dos-me-m1-m2-items.sql +!Docs/Proposals/2026-10-28-dos-me-m1-m2-items-rollback.sql +!Docs/Proposals/2026-10-28-dos-me-m1-m2-items-verify.sql +!Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey.sql +!Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-rollback.sql +!Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-verify.sql +!Docs/Proposals/2026-10-29-dos-me-item-rpcs.sql +!Docs/Proposals/2026-10-29-dos-me-item-rpcs-rollback.sql +!Docs/Proposals/2026-10-29-dos-me-m3-m4.sql +!Docs/Proposals/2026-10-29-dos-me-m3-m4-rollback.sql +!Docs/Proposals/2026-10-29-dos-me-m3-m4-verify.sql * - Copy* * - Bản sao* *.dump diff --git a/Docs/Proposals/2026-10-28-dos-me-m1-m2-items-rollback.sql b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items-rollback.sql new file mode 100644 index 000000000..9a9f6714b --- /dev/null +++ b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items-rollback.sql @@ -0,0 +1,230 @@ +-- PROPOSAL, NOT APPLIED. Rollback of 2026-10-28-dos-me-m1-m2-items.sql (DOS/Rate#1150, +-- design section 7 "M1"/"M2" reversibility). Run with the writers paused (importer and Rate +-- Services workers stopped), in one transaction. Re-runnable. +-- +-- Never TRUNCATE rate.items: it fails while the FKs exist, and TRUNCATE ... CASCADE would wipe +-- master_products, products and through them price history, reviews and alerts. +-- Data written only by this migration (Items, item_id, variant_key, variant_source) is lost; +-- nothing older depends on it. The two rating functions get their previous bodies back +-- (20261018153000 and 20261018120103), so the old readers keep working. + +BEGIN; + +-- 1. Triggers first, so the column drops below fire nothing. +DROP TRIGGER IF EXISTS trg_products_recheck_retired_variant ON rate.products; +DROP TRIGGER IF EXISTS trg_products_refresh_left ON rate.products; +DROP TRIGGER IF EXISTS trg_products_sync_item ON rate.products; +DROP TRIGGER IF EXISTS trg_master_products_move_listings ON rate.master_products; +DROP TRIGGER IF EXISTS trg_master_products_sync_item ON rate.master_products; +DROP TRIGGER IF EXISTS trg_rate_items_set_updated_at ON rate.items; + +-- 2. Previous rating bodies (verbatim from the reference migrations). +CREATE OR REPLACE FUNCTION rate.refresh_master_product_rating(p_master_id UUID) +RETURNS void +LANGUAGE sql +SECURITY DEFINER +SET search_path = '' +AS $$ + UPDATE rate.master_products m + SET user_rating_weighted = COALESCE(s.weighted, 0), + authentic_review_count = COALESCE(s.n, 0), + updated_at = now() + FROM ( + SELECT + round( + (sum(r.rating_star * r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user')) + / NULLIF(sum(r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user'), 0), + 2 + ) AS weighted, + (count(*) FILTER (WHERE r.reviewer_type = 'user'))::INTEGER AS n + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + JOIN rate.review_flags f ON f.review_id = r.id + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + -- Reviews excluded from the rating (unanalyzed community reviews, + -- flagged ones) never count, whatever their weight (20261014033000). + AND COALESCE(f.include_in_rating, true) + AND r.helpfulness_weight > 0 + ) s + WHERE m.id = p_master_id; +$$; + +CREATE OR REPLACE FUNCTION rate.refresh_review_vote_counts(p_review_id UUID) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_up INTEGER; + v_down INTEGER; + v_rep INTEGER; + v_cat TEXT; + v_verdict TEXT; + v_inc BOOLEAN; + v_master UUID; +BEGIN + SELECT + count(*) FILTER (WHERE vote = 'up'), + count(*) FILTER (WHERE vote = 'down'), + count(*) FILTER (WHERE reported) + INTO v_up, v_down, v_rep + FROM rate.review_votes + WHERE review_id = p_review_id; + + SELECT f.flag_category, f.verdict, COALESCE(f.include_in_rating, true) + INTO v_cat, v_verdict, v_inc + FROM rate.review_flags f + WHERE f.review_id = p_review_id; + + -- Votes cannot lift a review that is excluded from the rating + -- (20261014033000, relied on by 20261017120000's unanalyzed reviews). + UPDATE rate.reviews + SET upvotes = v_up, + downvotes = v_down, + report_count = v_rep, + helpfulness_weight = CASE + WHEN NOT COALESCE(v_inc, true) THEN 0.00 + ELSE rate.compute_review_weight( + v_up, v_down, v_rep, + COALESCE(v_cat, 'authentic'), + COALESCE(v_verdict, 'unanalyzed') + ) + END + WHERE id = p_review_id; + + SELECT p.cluster_id INTO v_master + FROM rate.reviews r + JOIN rate.products p ON p.id = r.product_id + WHERE r.id = p_review_id; + + IF v_master IS NOT NULL THEN + PERFORM rate.refresh_master_product_rating(v_master); + END IF; +END; +$$; + +CREATE OR REPLACE FUNCTION rate.refresh_master_market_stats(p_master_id UUID) +RETURNS void +LANGUAGE sql +SECURITY DEFINER +SET search_path = '' +AS $$ + WITH priced AS ( + SELECT + p.price_current AS price, + COALESCE(NULLIF(p.price_max, 0), p.price_current) AS price_top, + p.merchant_id, + (pg_catalog.row_number() OVER (ORDER BY p.price_current, p.id) - 1)::INTEGER AS k, + (pg_catalog.count(*) OVER ())::INTEGER AS m + FROM rate.products p + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + AND p.availability = 'in_stock' + AND p.price_current > 0 + ), + judged AS ( + SELECT + c.price, + c.price_top, + c.merchant_id, + (c.m >= 3 AND c.price < 0.4 * r.price) AS implausible + FROM priced c + LEFT JOIN priced r + ON r.k = CASE WHEN (c.m - 2) / 2 < c.k THEN (c.m - 2) / 2 ELSE (c.m - 2) / 2 + 1 END + ), + s AS ( + SELECT + MIN(j.price) FILTER (WHERE NOT j.implausible) AS min_price, + MAX(j.price_top) AS max_price, + COUNT(DISTINCT j.merchant_id)::INTEGER AS merchant_count + FROM judged j + ) + UPDATE rate.master_products m + SET min_price = s.min_price, + max_price = s.max_price, + merchant_count = s.merchant_count, + updated_at = now() + FROM s + WHERE m.id = p_master_id + AND (m.min_price, m.max_price, m.merchant_count) + IS DISTINCT FROM (s.min_price, s.max_price, s.merchant_count); +$$; + +-- 2b. Put back what retire_item_unknown_variant changed (it records it in `metadata`), then +-- recompute every variant's rating and prices with the old bodies, so no pooled Item rating +-- stays mirrored on a variant. Listings that never had a variant (written with +-- RATE_ITEM_MODEL=nullable_variant) keep cluster_id NULL: the old schema cannot say which +-- Item they belong to. Count them first with +-- SELECT count(*) FROM rate.products WHERE cluster_id IS NULL AND item_id IS NOT NULL +-- AND NOT (metadata ? 'retired_unknown_variant'); +UPDATE rate.master_products mp +SET status = mp.metadata ->> 'status_before_retire', + metadata = mp.metadata - 'status_before_retire' - 'retired_by' +WHERE mp.metadata ->> 'retired_by' = 'retire_item_unknown_variant' + AND mp.metadata ? 'status_before_retire'; +-- Only while the item_id columns still exist (a second run finds them dropped). +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_schema = 'rate' AND table_name = 'products' AND column_name = 'item_id') THEN + EXECUTE $q$ + UPDATE rate.products p + SET cluster_id = mp.id, + metadata = p.metadata - 'retired_unknown_variant' - 'variant_source_before_retire' + FROM rate.master_products mp + WHERE p.cluster_id IS NULL + AND p.metadata ? 'retired_unknown_variant' + AND mp.id = (p.metadata ->> 'retired_unknown_variant')::uuid + AND mp.item_id IS NOT DISTINCT FROM p.item_id + $q$; + END IF; +END $$; +DO $$ +DECLARE + v_id uuid; +BEGIN + FOR v_id IN SELECT m.id FROM rate.master_products m ORDER BY m.id LOOP + PERFORM rate.refresh_master_market_stats(v_id); + PERFORM rate.refresh_master_product_rating(v_id); + END LOOP; +END $$; + +-- 3. New functions. +DROP FUNCTION IF EXISTS rate.retire_item_unknown_variant(uuid); +DROP FUNCTION IF EXISTS rate.match_item(text, text, text, real); +DROP FUNCTION IF EXISTS rate.refresh_item_stats(uuid); +DROP FUNCTION IF EXISTS rate.refresh_item_rating(uuid); +DROP FUNCTION IF EXISTS rate.products_recheck_retired_variant(); +DROP FUNCTION IF EXISTS rate.products_refresh_left(); +DROP FUNCTION IF EXISTS rate.products_sync_item(); +DROP FUNCTION IF EXISTS rate.master_products_move_listings(); +DROP FUNCTION IF EXISTS rate.master_products_sync_item(); +DROP FUNCTION IF EXISTS rate.item_name_from_master(text); +DROP FUNCTION IF EXISTS rate.item_slug_from_master(text, text); + +-- 4. Constraints, indexes and columns on the existing tables. +ALTER TABLE rate.products DROP CONSTRAINT IF EXISTS fk_products_item; +ALTER TABLE rate.products DROP CONSTRAINT IF EXISTS products_variant_source_check; +ALTER TABLE rate.master_products DROP CONSTRAINT IF EXISTS fk_master_products_item; +DO $$ +BEGIN + IF to_regclass('rate.items') IS NOT NULL THEN + ALTER TABLE rate.items DROP CONSTRAINT IF EXISTS fk_items_default_variant; + END IF; +END $$; +DROP INDEX IF EXISTS rate.idx_rate_products_item; +DROP INDEX IF EXISTS rate.idx_master_products_item; +DROP INDEX IF EXISTS rate.uq_master_products_item_variant; +ALTER TABLE rate.products DROP COLUMN IF EXISTS variant_source; +ALTER TABLE rate.products DROP COLUMN IF EXISTS item_id; +ALTER TABLE rate.master_products DROP COLUMN IF EXISTS sort_order; +ALTER TABLE rate.master_products DROP COLUMN IF EXISTS variant_key; +ALTER TABLE rate.master_products DROP COLUMN IF EXISTS item_id; + +-- 5. The Item table (its policies, grants and indexes go with it). +DROP TABLE IF EXISTS rate.items; + +COMMIT; diff --git a/Docs/Proposals/2026-10-28-dos-me-m1-m2-items-verify.sql b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items-verify.sql new file mode 100644 index 000000000..ab1f37966 --- /dev/null +++ b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items-verify.sql @@ -0,0 +1,90 @@ +-- PROPOSAL. Read-only verification for 2026-10-28-dos-me-m1-m2-items.sql (DOS/Rate#1150, +-- design section 7.3). Run before (inventory), after M1+M2, and again as the reconciliation +-- after the last writer ships Item-aware code. Every query is a SELECT. Expected results are +-- in the trailing comments; `scripts/test-item-migration-proposal.mjs` runs them on the +-- replay stack and fails on any non-expected value. + +-- V1 Every variant has an Item. expect 0 +SELECT count(*) AS masters_without_item FROM rate.master_products WHERE item_id IS NULL; + +-- V2 Every listing on a variant has an Item. expect 0 +SELECT count(*) AS listings_without_item +FROM rate.products WHERE item_id IS NULL AND cluster_id IS NOT NULL; + +-- V3 A listing's Item is its variant's Item. expect 0 +SELECT count(*) AS listing_item_mismatch +FROM rate.products p JOIN rate.master_products mp ON mp.id = p.cluster_id +WHERE p.item_id IS DISTINCT FROM mp.item_id; + +-- V4 variant_key is the key's third segment. expect 0 +SELECT count(*) AS variant_key_drift +FROM rate.master_products +WHERE variant_key IS DISTINCT FROM NULLIF(split_part(model_key, ':', 3), ''); + +-- V5 An Item's key is its variants' brand:model. expect 0 +SELECT count(*) AS item_key_drift +FROM rate.master_products mp JOIN rate.items i ON i.id = mp.item_id +WHERE i.item_key <> split_part(mp.model_key, ':', 1) || ':' || split_part(mp.model_key, ':', 2); + +-- V6 A published variant never sits under an unpublished Item. expect 0 +SELECT count(*) AS published_variant_hidden +FROM rate.master_products mp JOIN rate.items i ON i.id = mp.item_id +WHERE mp.status = 'published' AND i.status <> 'published'; + +-- V7 Item "from" price = min over its published, priced variants. expect 0 +SELECT count(*) AS item_min_price_drift +FROM rate.items i +WHERE i.min_price IS DISTINCT FROM ( + SELECT COALESCE(min(mp.min_price) FILTER (WHERE mp.min_price > 0), 0) + FROM rate.master_products mp + WHERE mp.item_id = i.id AND mp.status = 'published'); + +-- V8 Item listing counts. expect 0 +SELECT count(*) AS item_count_drift +FROM rate.items i +CROSS JOIN LATERAL ( + SELECT count(*)::integer AS listings, + (count(*) FILTER (WHERE p.cluster_id IS NULL))::integer AS unknown + FROM rate.products p WHERE p.item_id = i.id AND p.origin = 'crawl' +) c +WHERE (i.listing_count, i.unknown_variant_listing_count) IS DISTINCT FROM (c.listings, c.unknown); + +-- V9 An Item slug may only equal one of its own variants' slugs. expect 0 +SELECT count(*) AS foreign_slug_clash +FROM rate.items i JOIN rate.master_products mp ON mp.slug = i.slug +WHERE mp.item_id IS DISTINCT FROM i.id; + +-- V10 Item names and slugs carry no storage. expect 0 +SELECT count(*) AS sized_item_names +FROM rate.items +WHERE slug ~ '-[0-9]+(gb|tb)$' OR name ~* '[[:space:]][0-9]+[[:space:]]?(gb|tb)$'; + +-- V11 Item specs carry no storage or RAM (variant overrides). expect 0 +SELECT count(*) AS sized_item_specs +FROM rate.items +WHERE specifications #> '{hardware,storage_gb}' IS NOT NULL + OR specifications #> '{hardware,ram_gb}' IS NOT NULL + OR key_specs ? 'storage' OR key_specs ? 'ram'; + +-- V12 The variant rating mirrors its Item's. expect 0 +SELECT count(*) AS rating_mirror_drift +FROM rate.master_products mp JOIN rate.items i ON i.id = mp.item_id +WHERE (COALESCE(mp.user_rating_weighted, 0), COALESCE(mp.authentic_review_count, 0)) + IS DISTINCT FROM (COALESCE(i.user_rating_weighted, 0), i.authentic_review_count); + +-- V13 variant_source values (information): legacy after the backfill, then evidence values. +SELECT COALESCE(variant_source, '(null)') AS variant_source, count(*) +FROM rate.products GROUP BY 1 ORDER BY 1; + +-- V14 Multi-size Items still holding a live unknown variant (retire candidates). expect 0 +-- after the importer's retire call; information before (list them with i.item_key). +SELECT count(*) AS retire_candidates +FROM rate.items i +JOIN rate.master_products u ON u.item_id = i.id AND u.variant_key = 'unknown' AND u.status <> 'archived' +WHERE EXISTS (SELECT 1 FROM rate.master_products s + WHERE s.item_id = i.id AND s.status <> 'archived' AND s.variant_key ~ '^[0-9]+(gb|tb)$'); + +-- V15 Published variant counts per Item, for the page smoke test (information). +SELECT i.slug, count(mp.id) AS variants +FROM rate.items i JOIN rate.master_products mp ON mp.item_id = i.id +WHERE i.status = 'published' GROUP BY 1 ORDER BY 2 DESC, 1; diff --git a/Docs/Proposals/2026-10-28-dos-me-m1-m2-items.md b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items.md new file mode 100644 index 000000000..55bdd4f62 --- /dev/null +++ b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items.md @@ -0,0 +1,64 @@ +# Proposal: DOS-Me migration M1 + M2 for Items (DOS/Rate#1150) + +**Status:** proposal for JOY, **not applied**. Zero DDL on the shared prod Supabase from DOS/Rate; JOY reviews and applies it through DOS-Me (preview, then beta, then prod). The DOS-Me beta flow is still locked (ROADMAP 6.4). + +Design: [`2026-10-27-item-variant-listing-model.md`](./2026-10-27-item-variant-listing-model.md), sections 4.3, 4.5, 7, 7.1, 7.3 (approved by JOY on 2026-10-27, #1129). + +## Files + +| File | What | +| :------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`2026-10-28-dos-me-m1-m2-items.sql`](./2026-10-28-dos-me-m1-m2-items.sql) | M1 (schema, RLS, grants, sync triggers, race guards) and M2 (RPCs, backfill) in one transaction. Idempotent. | +| [`2026-10-28-dos-me-m1-m2-items-rollback.sql`](./2026-10-28-dos-me-m1-m2-items-rollback.sql) | Full rollback, re-runnable; restores the two rating functions' previous bodies. Writers paused. | +| [`2026-10-28-dos-me-m1-m2-items-verify.sql`](./2026-10-28-dos-me-m1-m2-items-verify.sql) | Read-only checks (V1 to V15) to run before, after, and as the reconciliation. | +| [`scripts/test-item-migration-proposal.mjs`](../../scripts/test-item-migration-proposal.mjs) | CI test on the throwaway PostgreSQL 17 stack of the migration replay: proposal twice, verification, triggers and RPCs as `anon` / `rate_worker` / `service_role`, rollback twice, re-apply. | + +## Order of operations + +1. **M0 first: [`2026-10-29-dos-me-m0-seed-rekey.sql`](./2026-10-29-dos-me-m0-seed-rekey.sql)** (DOS/Rate#1212, #1123: re-key of the six drifted seed masters and the seed change right after, with JOY's approval), so the Items get the normalizer's Item keys (`apple:watch-ultra-3`, `apple:airpods-pro-2`). +2. This migration on preview, run the verification file, then beta, then prod. +3. The importer keeps `RATE_ITEM_MODEL` off until Rate Services handles NULL `cluster_id` (design task 13). Flag `item` can come first; `nullable_variant` after task 13. +4. Mirror the applied file into `supabase/migrations/` and regenerate `database.types.ts` (as #1147 did). + +## Contents and the contracts they satisfy + +| Piece | Contract / source | +| :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rate.items` (design 4.3 columns), RLS: public reads published, `service_role` all, `rate_worker` reads all and updates spec columns only (description, specifications, key_specs, spec_provenance, images, thumbnail, metadata), never status, key, prices or counts | matcher reads draft Items (#1179); seed spec sync writes as service role (#1208); Rate Services itself only SELECTs `id, item_key, name` and EXECUTEs `refresh_item_stats` (RS PO, RS#700); the grants are sized for our engine when Rate Services runs it as `rate_worker`: `syncArenaSpecsToItem` (#1213) updates `specifications, key_specs, updated_at` guarded by `updated_at`, ingest writes `products.item_id / variant_source` (table-level grant on products and master_products, so the new columns are covered; no column-scoped SELECT exists on them), the matcher EXECUTEs `match_item`. All asserted in the CI replay | +| `master_products.item_id`, `variant_key`, `sort_order`; `products.item_id`, `variant_source` (+ CHECK) | pipeline writes (#1179, #1190) | +| index `rate.products(item_id)` | Rate Services filters by Item (RS#688, PR #689) | +| M1 sync triggers: a master inserted or re-keyed gets its Item from its key (draft, published with its variant); its listings follow; a listing's Item follows its variant | design 4.3 compatibility sync; `itemIdOfMaster` in the matcher fails loudly without it | +| `match_item(search_title, p_brand, p_model, p_threshold)` -> `item_id, item_key, name, similarity` | matcher Item Tier 2 (#1179) | +| `refresh_item_stats(p_item_id)`: prices from published variants, listing / unknown-variant / merchant counts, **and the pooled Item rating** mirrored onto the variants | pipeline and review analyzer (#1190, #1208) | +| `refresh_master_product_rating(p_master_id)`: same signature, rates the whole Item when the master has one | design 4.5 | +| `refresh_review_vote_counts`: resolves the Item through `products.item_id`, so votes on a listing with no variant count | design 4.5 | +| `retire_item_unknown_variant(p_item_id)`: one transaction, Item row locked; listings of a multi-size Item's live `unknown` variant move to no variant, the variant is archived, stats refreshed; earbuds never | Codex P1 left open on #1179 | +| `products` BEFORE trigger: a `manual` variant is never moved by a later write (an ingest that read the row before it was hand-set included); admin tools move it with `SET LOCAL rate.variant_override = 'on'` | Codex P1 on #1179 (read-then-upsert race) | +| `products` AFTER trigger: the Item and variant a listing leaves are refreshed by the statement that moves it, under the row lock | Codex finding on #1190 (overlapping ingests leave the middle Item stale) | +| Backfill (design 7.1) with an up-front slug-collision abort; Item specs and key specs without storage and RAM | design 7.1, #1208 rule "no storage on an Item" | + +## Concurrency and compatibility + +- **One lock order everywhere: listing, then variant, then Item.** The listing triggers refresh the left variant before the left Item and take no variant lock themselves (an `INSERT ... ON CONFLICT` runs the BEFORE trigger before it locks the conflicting listing); the rating refresh mirrors onto the variants in id order with `SKIP LOCKED` (a skipped variant belongs to a concurrent refresh of the same Item, which mirrors itself), then writes the Item; the retire RPC locks the listings, then the variant, then refreshes the Item. +- **A retired `unknown` variant takes no listing.** The listing trigger turns a write onto a retired `unknown` into "no variant, same Item". A write that races a running retire is caught by the retire's second sweep (writes committed before it) or by a deferred constraint trigger that re-checks the variant at the writer's commit (the writer waits on the variant's FK lock until the retire commits, so it then sees the variant archived). Calling the RPC again on an Item whose `unknown` is already retired also sweeps any straggler. +- **Known trade-off: the variant rating mirror can lag.** To avoid sibling-variant deadlocks the mirror skips a variant another transaction holds (`SKIP LOCKED`). If that transaction does not itself refresh the Item (for example a specs update), the variant keeps its previous mirrored rating until the Item's next refresh (any ingest, vote or retire on it). The Item's own rating is always current; only old readers of the variant columns see the lag. V12 reports it; the reconciliation `SELECT rate.refresh_item_rating(id) FROM rate.items` clears it. +- **Old writers keep Item numbers fresh.** `refresh_master_market_stats(p_master_id)` keeps its signature and query and also refreshes the variant's Item (removed with the sync triggers in M4), so the importer with the flag off and Rate Services before task 13 do not leave Item prices or counts stale. +- **The retire RPC records what it changed** in `metadata` (listing: `retired_unknown_variant`, `variant_source_before_retire`; variant: `status_before_retire`, `retired_by`), so the rollback can put those listings back on their variant. Any later variant or Item change on the listing clears its markers, and the rollback restores a listing only onto a variant of its own Item. +- **`rate.variant_override` is a cooperative switch, not a permission.** Any role that can update `rate.products` could set it; only the worker and service role can, and both are trusted writers. The retire RPC restores the caller's value. +- **An Item is never demoted automatically.** Publishing a variant publishes its Item; archiving every variant leaves the Item's status to an admin (or M3/M4 tooling). + +## Decisions to confirm (JOY) + +- **`match_item` filters by brand, not by model.** The design names a `p_model` argument; filtering by it would only find the exact model, which Tier 1 already did. The matcher only links a fuzzy Item hit to variants that exist, so a looser match cannot create a variant under a foreign Item. +- **Item rating with no reviews is stored as 0**, like the variant columns today (`refresh_master_product_rating` writes `COALESCE(..., 0)`). The web shows "no rating" when the review count is 0. +- **`rate.v_products_with_merchant_trust` is not changed.** The design suggested appending `item_id`; doing so breaks a re-run of `20261018153000` (see the note in `20261020230000`). Readers take `item_id` from `rate.products`. +- **Price alerts on a retired `unknown` variant stay on it** (archived): an alert needs a size. M3's report lists them (design 7.3). +- **Admin override setting name:** `rate.variant_override`. Any admin tool that re-assigns a hand-set variant must set it locally. + +## Not in this migration + +Web RPCs (`search_items_v1`, facet and autocomplete v2, `resolve_catalog_slug`, `submit_community_review` by Item) belong to the web tasks (design section 10, tasks 7 to 11). M3 (bulk retire with a backup table) and M4 (NOT NULL, guard triggers, dropping the sync triggers) are later phases. + +## Rollback + +Run `2026-10-28-dos-me-m1-m2-items-rollback.sql` with the importer and Rate Services workers paused. It drops the triggers, the new functions, the new columns and `rate.items`, and restores `refresh_master_product_rating` (20261018153000) and `refresh_review_vote_counts` (20261018120103). Never `TRUNCATE rate.items`. diff --git a/Docs/Proposals/2026-10-28-dos-me-m1-m2-items.sql b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items.sql new file mode 100644 index 000000000..2cc370d2d --- /dev/null +++ b/Docs/Proposals/2026-10-28-dos-me-m1-m2-items.sql @@ -0,0 +1,869 @@ +-- PROPOSAL, NOT APPLIED. DOS-Me migration M1 + M2 for the Item / variant / listing model +-- (DOS/Rate#1150; design Docs/Proposals/2026-10-27-item-variant-listing-model.md, sections +-- 4.3, 4.5, 7, 7.1). JOY reviews and applies it through DOS-Me (preview, then beta, then +-- prod). Zero DDL on prod from DOS/Rate. Rollback: 2026-10-28-dos-me-m1-m2-items-rollback.sql. +-- Verification: 2026-10-28-dos-me-m1-m2-items-verify.sql. +-- +-- Order: M0 (2026-10-29-dos-me-m0-seed-rekey.sql, DOS/Rate#1212 / #1123) runs first, so the Items +-- are created with the normalizer's Item keys. +-- +-- Idempotent: every step is IF NOT EXISTS, guarded by pg_constraint / pg_trigger, CREATE OR +-- REPLACE, or a re-runnable UPDATE ... WHERE . One transaction (M1 and M2 ship +-- together so no Item is created by a trigger and then skipped by the backfill). +-- +-- Contracts the importer already calls (DOS/Rate#1179, #1190, #1208, #1202): +-- rate.items(id, item_key, status, description, specifications, key_specs, images, ...) +-- rate.master_products.item_id / variant_key, rate.products.item_id / variant_source +-- rate.match_item(search_title, p_brand, p_model, p_threshold) +-- -> item_id, item_key, name, similarity +-- rate.refresh_item_stats(p_item_id): prices, counts AND the Item's pooled rating +-- rate.refresh_master_product_rating(p_master_id): same signature, Item-aware body +-- M1 sync triggers: a master gets its Item from its key; a listing's item follows its variant +-- Added here for the matcher's open Codex P1s: +-- rate.retire_item_unknown_variant(p_item_id): transactional unknown-variant retire +-- a products trigger that keeps a hand-set (manual) variant against a racing ingest, and +-- an AFTER trigger that refreshes the Item and variant a listing leaves (overlapping ingests) +-- +-- Not in this migration: rate.v_products_with_merchant_trust is left unchanged (appending a +-- column makes a re-run of 20261018153000 fail, see 20261020230000); listings read item_id +-- from rate.products. Web RPCs (search_items_v1, facets/autocomplete v2, resolve_catalog_slug, +-- submit_community_review by Item) belong to the web tasks (design section 10, tasks 7 to 11). + +BEGIN; + +-- M1.1 rate.items -------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS rate.items ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + item_key text NOT NULL, + slug text NOT NULL, + name text NOT NULL, + brand text NOT NULL, + model text NOT NULL, + category_id uuid REFERENCES rate.categories(id) ON DELETE SET NULL, + status text NOT NULL DEFAULT 'draft' + CHECK (status IN ('draft', 'published', 'archived')), + entity_type text NOT NULL DEFAULT 'product', + description text, + specifications jsonb NOT NULL DEFAULT '{}'::jsonb, + key_specs jsonb NOT NULL DEFAULT '{}'::jsonb, + spec_provenance jsonb NOT NULL DEFAULT '{}'::jsonb, + images text[] NOT NULL DEFAULT '{}', + thumbnail text, + default_variant_id uuid, + min_price numeric(15, 2) NOT NULL DEFAULT 0, + max_price numeric(15, 2) NOT NULL DEFAULT 0, + price_variant_count integer NOT NULL DEFAULT 0, + merchant_count integer NOT NULL DEFAULT 0, + listing_count integer NOT NULL DEFAULT 0, + unknown_variant_listing_count integer NOT NULL DEFAULT 0, + total_reviews integer NOT NULL DEFAULT 0, + average_rating numeric(3, 2) NOT NULL DEFAULT 0, + user_rating_weighted numeric(3, 2), + authentic_review_count integer NOT NULL DEFAULT 0, + expert_rating numeric(3, 2), + expert_review_summary text, + expert_criteria_scores jsonb NOT NULL DEFAULT '{}'::jsonb, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT items_item_key_key UNIQUE (item_key), + CONSTRAINT items_slug_key UNIQUE (slug) +); +-- No shape CHECK on item_key: legacy keys still get an Item (design 7.1). + +-- M1.2 columns on the variant and listing tables ------------------------------------------ + +ALTER TABLE rate.master_products ADD COLUMN IF NOT EXISTS item_id uuid; +ALTER TABLE rate.master_products ADD COLUMN IF NOT EXISTS variant_key text; +ALTER TABLE rate.master_products ADD COLUMN IF NOT EXISTS sort_order smallint; +ALTER TABLE rate.products ADD COLUMN IF NOT EXISTS item_id uuid; +ALTER TABLE rate.products ADD COLUMN IF NOT EXISTS variant_source text; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass AND conname = 'products_variant_source_check') THEN + ALTER TABLE rate.products ADD CONSTRAINT products_variant_source_check CHECK + (variant_source IS NULL OR variant_source IN ('title', 'sku', 'single_size', 'single_variant', 'manual', 'legacy')); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.master_products'::regclass AND conname = 'fk_master_products_item') THEN + ALTER TABLE rate.master_products ADD CONSTRAINT fk_master_products_item + FOREIGN KEY (item_id) REFERENCES rate.items(id) ON DELETE RESTRICT; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass AND conname = 'fk_products_item') THEN + ALTER TABLE rate.products ADD CONSTRAINT fk_products_item + FOREIGN KEY (item_id) REFERENCES rate.items(id) ON DELETE RESTRICT; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.items'::regclass AND conname = 'fk_items_default_variant') THEN + ALTER TABLE rate.items ADD CONSTRAINT fk_items_default_variant + FOREIGN KEY (default_variant_id) REFERENCES rate.master_products(id) ON DELETE SET NULL; + END IF; +END $$; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_master_products_item_variant + ON rate.master_products (item_id, variant_key) WHERE item_id IS NOT NULL AND variant_key IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_master_products_item ON rate.master_products (item_id); +-- Rate Services filters listings by Item (RS#688, PR #689). +CREATE INDEX IF NOT EXISTS idx_rate_products_item ON rate.products (item_id); +CREATE INDEX IF NOT EXISTS idx_items_status_cat_price ON rate.items (status, category_id, min_price); +CREATE INDEX IF NOT EXISTS idx_items_status_rating ON rate.items (status, user_rating_weighted DESC); + +-- pg_trgm lives in schema `sign` on prod and elsewhere in other stacks: look it up at run time. +DO $$ +DECLARE + v_trgm text; +BEGIN + SELECT n.nspname INTO v_trgm + FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'pg_trgm'; + IF v_trgm IS NULL THEN + RAISE EXCEPTION 'pg_trgm extension is not installed'; + END IF; + EXECUTE format( + 'CREATE INDEX IF NOT EXISTS idx_items_name_trgm ON rate.items USING gin (name %I.gin_trgm_ops)', + v_trgm + ); +END $$; + +-- M1.3 RLS, grants, updated_at ------------------------------------------------------------- + +ALTER TABLE rate.items ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS "Public can view published items" ON rate.items; +CREATE POLICY "Public can view published items" ON rate.items + FOR SELECT USING (status = 'published'); +DROP POLICY IF EXISTS "Service role write items" ON rate.items; +CREATE POLICY "Service role write items" ON rate.items + FOR ALL TO service_role USING (true) WITH CHECK (true); +-- The worker reads every Item (the matcher links listings to draft Items too); it writes +-- Items only through the SECURITY DEFINER triggers and RPCs below. +DROP POLICY IF EXISTS "Rate worker read items" ON rate.items; +CREATE POLICY "Rate worker read items" ON rate.items + FOR SELECT TO rate_worker USING (true); + +GRANT SELECT ON rate.items TO anon, authenticated; +GRANT ALL ON rate.items TO service_role; +GRANT SELECT ON rate.items TO rate_worker; +-- Rate Services updates Item specs before the flag goes on (RS#700): spec and description +-- columns only. Identity, status, prices, counts and ratings stay with the triggers and +-- RPCs, so the worker cannot publish or re-key an Item. +REVOKE UPDATE ON rate.items FROM rate_worker; +GRANT UPDATE (description, specifications, key_specs, spec_provenance, images, thumbnail, + metadata, updated_at) + ON rate.items TO rate_worker; +DROP POLICY IF EXISTS "Rate worker update items" ON rate.items; +CREATE POLICY "Rate worker update items" ON rate.items + FOR UPDATE TO rate_worker USING (true) WITH CHECK (true); + +DROP TRIGGER IF EXISTS trg_rate_items_set_updated_at ON rate.items; +CREATE TRIGGER trg_rate_items_set_updated_at + BEFORE UPDATE ON rate.items + FOR EACH ROW EXECUTE FUNCTION rate.set_updated_at_timestamp(); + +-- M1.4 Item slug and name from a variant (design 7.1) --------------------------------------- + +CREATE OR REPLACE FUNCTION rate.item_slug_from_master(p_slug text, p_model_key text) +RETURNS text +LANGUAGE sql +IMMUTABLE +SET search_path = '' +AS $$ + SELECT COALESCE( + NULLIF(regexp_replace(COALESCE(p_slug, ''), '-([0-9]+(gb|tb)|standard|unknown)$', ''), ''), + trim(BOTH '-' FROM regexp_replace( + lower(split_part(COALESCE(p_model_key, ''), ':', 1) || '-' || split_part(COALESCE(p_model_key, ''), ':', 2)), + '[^a-z0-9-]+', '-', 'g')) + ); +$$; + +CREATE OR REPLACE FUNCTION rate.item_name_from_master(p_name text) +RETURNS text +LANGUAGE sql +IMMUTABLE +SET search_path = '' +AS $$ + SELECT COALESCE( + NULLIF(btrim(regexp_replace(COALESCE(p_name, ''), '[[:space:]]+[0-9]+([.,][0-9]+)?[[:space:]]?(gb|tb)$', '', 'i')), ''), + COALESCE(p_name, '') + ); +$$; + +REVOKE EXECUTE ON FUNCTION rate.item_slug_from_master(text, text) FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.item_name_from_master(text) FROM PUBLIC, anon, authenticated; + +-- M1.5 compatibility sync (removed in M4) -------------------------------------------------- + +-- A master inserted or re-keyed without its Item gets one from its key (created as draft); +-- publishing a variant publishes its Item. SECURITY DEFINER: rate_worker holds no write grant +-- on rate.items; the function only derives an Item from the row being written. +CREATE OR REPLACE FUNCTION rate.master_products_sync_item() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_key text; + v_slug text; +BEGIN + IF TG_OP = 'INSERT' OR NEW.model_key IS DISTINCT FROM OLD.model_key OR NEW.item_id IS NULL THEN + v_key := split_part(NEW.model_key, ':', 1) || ':' || split_part(NEW.model_key, ':', 2); + NEW.variant_key := NULLIF(split_part(NEW.model_key, ':', 3), ''); + NEW.item_id := NULL; + SELECT i.id INTO NEW.item_id FROM rate.items i WHERE i.item_key = v_key; + IF NEW.item_id IS NULL THEN + v_slug := rate.item_slug_from_master(NEW.slug, NEW.model_key); + BEGIN + INSERT INTO rate.items (item_key, slug, name, brand, model, category_id, status) + VALUES (v_key, v_slug, rate.item_name_from_master(NEW.name), NEW.brand, NEW.model, + NEW.category_id, CASE WHEN NEW.status = 'published' THEN 'published' ELSE 'draft' END) + ON CONFLICT (item_key) DO NOTHING; + EXCEPTION WHEN unique_violation THEN + -- The slug belongs to another Item: suffix it with a hash of the key. + INSERT INTO rate.items (item_key, slug, name, brand, model, category_id, status) + VALUES (v_key, left(v_slug, 190) || '-' || left(md5(v_key), 8), + rate.item_name_from_master(NEW.name), NEW.brand, NEW.model, + NEW.category_id, CASE WHEN NEW.status = 'published' THEN 'published' ELSE 'draft' END) + ON CONFLICT (item_key) DO NOTHING; + END; + SELECT i.id INTO NEW.item_id FROM rate.items i WHERE i.item_key = v_key; + END IF; + END IF; + IF NEW.status = 'published' AND NEW.item_id IS NOT NULL THEN + UPDATE rate.items SET status = 'published' WHERE id = NEW.item_id AND status <> 'published'; + END IF; + RETURN NEW; +END $$; + +-- A master's listings follow it to a new Item in the same statement. +CREATE OR REPLACE FUNCTION rate.master_products_move_listings() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + IF NEW.item_id IS DISTINCT FROM OLD.item_id THEN + UPDATE rate.products SET item_id = NEW.item_id + WHERE cluster_id = NEW.id AND item_id IS DISTINCT FROM NEW.item_id; + END IF; + RETURN NULL; +END $$; + +-- A listing's Item follows its variant whenever a variant is set (NULL variant: keep the +-- Item the writer supplied). A hand-set variant (variant_source 'manual') is evidence: a +-- later write, including an ingest that read the listing before it was hand-set, never +-- moves it (the read-then-upsert race of DOS/Rate#1179). Admin tools and the retire RPC move +-- it on purpose with SET LOCAL rate.variant_override = 'on'. +CREATE OR REPLACE FUNCTION rate.products_sync_item() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_status text; + v_variant_key text; +BEGIN + IF TG_OP = 'UPDATE' + AND OLD.variant_source = 'manual' + AND COALESCE(current_setting('rate.variant_override', true), '') <> 'on' + AND (NEW.cluster_id IS DISTINCT FROM OLD.cluster_id + OR NEW.variant_source IS DISTINCT FROM OLD.variant_source + OR NEW.item_id IS DISTINCT FROM OLD.item_id) THEN + NEW.cluster_id := OLD.cluster_id; + NEW.variant_source := OLD.variant_source; + NEW.item_id := OLD.item_id; + END IF; + IF NEW.cluster_id IS NOT NULL THEN + -- No lock here: an INSERT ... ON CONFLICT runs this before it locks the conflicting + -- listing, so locking the variant would invert the listing -> variant lock order. + -- A write that races a running retire is swept up by the retire itself (it re-sweeps + -- after archiving, and a later call sweeps a retired variant again). + SELECT mp.item_id, mp.status, mp.variant_key INTO NEW.item_id, v_status, v_variant_key + FROM rate.master_products mp WHERE mp.id = NEW.cluster_id; + IF v_status = 'archived' AND v_variant_key = 'unknown' + AND (TG_OP = 'INSERT' OR NEW.cluster_id IS DISTINCT FROM OLD.cluster_id) THEN + -- A retired `unknown` variant takes no listing: the listing keeps the Item only. + NEW.cluster_id := NULL; + NEW.variant_source := NULL; + END IF; + END IF; + -- The retire markers describe one move; any later variant or Item change voids them, so + -- the rollback never puts a listing back on a variant of another Item. + IF TG_OP = 'UPDATE' AND NEW.metadata ? 'retired_unknown_variant' + AND (NEW.cluster_id IS NOT NULL OR NEW.item_id IS DISTINCT FROM OLD.item_id) THEN + NEW.metadata := NEW.metadata - 'retired_unknown_variant' - 'variant_source_before_retire'; + END IF; + RETURN NEW; +END $$; + +-- The Item and variant a listing leaves are refreshed by the statement that moves it, under +-- the row lock, so two overlapping ingests of one listing cannot leave the middle Item stale +-- (DOS/Rate#1185 / PR #1190 review). The writer still refreshes the new Item and variant. +CREATE OR REPLACE FUNCTION rate.products_refresh_left() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + -- Lock order everywhere: listing, then variant, then Item. The variant first also gives + -- the Item its post-move variant prices. + IF OLD.cluster_id IS NOT NULL AND OLD.cluster_id IS DISTINCT FROM NEW.cluster_id THEN + PERFORM rate.refresh_master_market_stats(OLD.cluster_id); + END IF; + IF OLD.item_id IS NOT NULL AND OLD.item_id IS DISTINCT FROM NEW.item_id THEN + PERFORM rate.refresh_item_stats(OLD.item_id); + END IF; + RETURN NULL; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.master_products_sync_item() FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.master_products_move_listings() FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.products_sync_item() FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.products_refresh_left() FROM PUBLIC, anon, authenticated; + +-- Re-check at commit: a write that read the variant before a running retire archived it +-- (its BEFORE trigger saw it live) waits on the variant's FK lock until the retire commits, +-- so at its own commit this deferred check sees the variant archived and moves the listing +-- to "no variant, same Item". The retire's sweeps cover writes committed before it. +CREATE OR REPLACE FUNCTION rate.products_recheck_retired_variant() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_override text := COALESCE(current_setting('rate.variant_override', true), ''); +BEGIN + IF NEW.cluster_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM rate.master_products mp + WHERE mp.id = NEW.cluster_id AND mp.status = 'archived' AND mp.variant_key = 'unknown') THEN + PERFORM set_config('rate.variant_override', 'on', true); + UPDATE rate.products p SET cluster_id = NULL, variant_source = NULL + WHERE p.id = NEW.id AND p.cluster_id = NEW.cluster_id; + PERFORM set_config('rate.variant_override', v_override, true); + END IF; + RETURN NULL; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.products_recheck_retired_variant() FROM PUBLIC, anon, authenticated; + +-- M2.1 Item rating and stats --------------------------------------------------------------- + +-- The Item's pooled rating over every crawled listing of the Item, including listings with +-- no variant (design 4.1, Q4), mirrored onto each variant for old readers. Same review filter +-- as refresh_master_product_rating (20261018153000). Variants are written before the Item +-- (lock order: listing, variant, Item). +CREATE OR REPLACE FUNCTION rate.refresh_item_rating(p_item_id uuid) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_weighted numeric; + v_n integer; +BEGIN + SELECT + COALESCE(round( + (sum(r.rating_star * r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user')) + / NULLIF(sum(r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user'), 0), + 2 + ), 0), + COALESCE((count(*) FILTER (WHERE r.reviewer_type = 'user'))::integer, 0) + INTO v_weighted, v_n + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + JOIN rate.review_flags f ON f.review_id = r.id + WHERE p.item_id = p_item_id + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + AND COALESCE(f.include_in_rating, true) + AND r.helpfulness_weight > 0; + + -- Mirror onto the variants in id order. A variant another transaction holds is skipped: + -- that transaction is a refresh of the same Item (every variant write here goes through + -- refresh_master_market_stats or this function) and mirrors when it gets here, so two + -- refreshes of sibling variants never wait on each other. + UPDATE rate.master_products m + SET user_rating_weighted = v_weighted, + authentic_review_count = v_n, + updated_at = now() + WHERE m.id IN ( + SELECT mp.id FROM rate.master_products mp + WHERE mp.item_id = p_item_id + AND (mp.user_rating_weighted, mp.authentic_review_count) IS DISTINCT FROM (v_weighted, v_n) + ORDER BY mp.id + FOR UPDATE SKIP LOCKED + ); + + UPDATE rate.items i + SET user_rating_weighted = v_weighted, + authentic_review_count = v_n + WHERE i.id = p_item_id + AND (i.user_rating_weighted, i.authentic_review_count) IS DISTINCT FROM (v_weighted, v_n); +END $$; + +-- Item prices (from its published variants), counts (over every crawled listing, with and +-- without a variant) and its rating. +CREATE OR REPLACE FUNCTION rate.refresh_item_stats(p_item_id uuid) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + PERFORM rate.refresh_item_rating(p_item_id); + UPDATE rate.items i + SET min_price = COALESCE(v.min_price, 0), + max_price = COALESCE(v.max_price, 0), + price_variant_count = COALESCE(v.n, 0), + listing_count = l.listing_count, + unknown_variant_listing_count = l.unknown_count, + merchant_count = l.merchant_count + FROM ( + SELECT min(mp.min_price) FILTER (WHERE mp.min_price > 0) AS min_price, + max(mp.max_price) FILTER (WHERE mp.min_price > 0) AS max_price, + (count(*) FILTER (WHERE mp.min_price > 0))::integer AS n + FROM rate.master_products mp + WHERE mp.item_id = p_item_id AND mp.status = 'published' + ) v, + ( + SELECT count(*)::integer AS listing_count, + (count(*) FILTER (WHERE p.cluster_id IS NULL))::integer AS unknown_count, + (count(DISTINCT p.merchant_id))::integer AS merchant_count + FROM rate.products p + WHERE p.item_id = p_item_id AND p.origin = 'crawl' + ) l + WHERE i.id = p_item_id + AND (i.min_price, i.max_price, i.price_variant_count, i.listing_count, + i.unknown_variant_listing_count, i.merchant_count) + IS DISTINCT FROM (COALESCE(v.min_price, 0), COALESCE(v.max_price, 0), COALESCE(v.n, 0), + l.listing_count, l.unknown_count, l.merchant_count); +END $$; + +-- Same signature; a master with an Item rates the whole Item (design 4.5). Without an Item +-- (before the backfill) the 20261018153000 body runs unchanged. +CREATE OR REPLACE FUNCTION rate.refresh_master_product_rating(p_master_id uuid) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_item uuid; +BEGIN + SELECT m.item_id INTO v_item FROM rate.master_products m WHERE m.id = p_master_id; + IF v_item IS NOT NULL THEN + PERFORM rate.refresh_item_rating(v_item); + RETURN; + END IF; + UPDATE rate.master_products m + SET user_rating_weighted = COALESCE(s.weighted, 0), + authentic_review_count = COALESCE(s.n, 0), + updated_at = now() + FROM ( + SELECT + round( + (sum(r.rating_star * r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user')) + / NULLIF(sum(r.helpfulness_weight) FILTER (WHERE r.reviewer_type = 'user'), 0), + 2 + ) AS weighted, + (count(*) FILTER (WHERE r.reviewer_type = 'user'))::integer AS n + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + JOIN rate.review_flags f ON f.review_id = r.id + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + AND COALESCE(f.include_in_rating, true) + AND r.helpfulness_weight > 0 + ) s + WHERE m.id = p_master_id; +END $$; + +-- Votes on a listing with no variant count too: resolve the Item through products.item_id +-- (design 4.5). Body of 20261018120103 otherwise unchanged. +CREATE OR REPLACE FUNCTION rate.refresh_review_vote_counts(p_review_id uuid) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_up INTEGER; + v_down INTEGER; + v_rep INTEGER; + v_cat TEXT; + v_verdict TEXT; + v_inc BOOLEAN; + v_master UUID; + v_item UUID; +BEGIN + SELECT + count(*) FILTER (WHERE vote = 'up'), + count(*) FILTER (WHERE vote = 'down'), + count(*) FILTER (WHERE reported) + INTO v_up, v_down, v_rep + FROM rate.review_votes + WHERE review_id = p_review_id; + + SELECT f.flag_category, f.verdict, COALESCE(f.include_in_rating, true) + INTO v_cat, v_verdict, v_inc + FROM rate.review_flags f + WHERE f.review_id = p_review_id; + + UPDATE rate.reviews + SET upvotes = v_up, + downvotes = v_down, + report_count = v_rep, + helpfulness_weight = CASE + WHEN NOT COALESCE(v_inc, true) THEN 0.00 + ELSE rate.compute_review_weight( + v_up, v_down, v_rep, + COALESCE(v_cat, 'authentic'), + COALESCE(v_verdict, 'unanalyzed') + ) + END + WHERE id = p_review_id; + + SELECT p.cluster_id, p.item_id INTO v_master, v_item + FROM rate.reviews r + JOIN rate.products p ON p.id = r.product_id + WHERE r.id = p_review_id; + + IF v_item IS NOT NULL THEN + PERFORM rate.refresh_item_rating(v_item); + ELSIF v_master IS NOT NULL THEN + PERFORM rate.refresh_master_product_rating(v_master); + END IF; +END; +$$; + +-- Compatibility hook (removed in M4 with the sync triggers): writers that do not know Items +-- (the importer with RATE_ITEM_MODEL off, Rate Services before task 13) refresh only the +-- variant; its Item is refreshed in the same call. Variant query of 20261024060000 unchanged. +CREATE OR REPLACE FUNCTION rate.refresh_master_market_stats(p_master_id uuid) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_item uuid; +BEGIN + WITH priced AS ( + SELECT + p.price_current AS price, + COALESCE(NULLIF(p.price_max, 0), p.price_current) AS price_top, + p.merchant_id, + (pg_catalog.row_number() OVER (ORDER BY p.price_current, p.id) - 1)::INTEGER AS k, + (pg_catalog.count(*) OVER ())::INTEGER AS m + FROM rate.products p + WHERE p.cluster_id = p_master_id + AND p.origin = 'crawl' + AND p.availability = 'in_stock' + AND p.price_current > 0 + ), + judged AS ( + SELECT + c.price, + c.price_top, + c.merchant_id, + (c.m >= 3 AND c.price < 0.4 * r.price) AS implausible + FROM priced c + LEFT JOIN priced r + ON r.k = CASE WHEN (c.m - 2) / 2 < c.k THEN (c.m - 2) / 2 ELSE (c.m - 2) / 2 + 1 END + ), + s AS ( + SELECT + MIN(j.price) FILTER (WHERE NOT j.implausible) AS min_price, + MAX(j.price_top) AS max_price, + COUNT(DISTINCT j.merchant_id)::INTEGER AS merchant_count + FROM judged j + ) + UPDATE rate.master_products m + SET min_price = s.min_price, + max_price = s.max_price, + merchant_count = s.merchant_count, + updated_at = now() + FROM s + WHERE m.id = p_master_id + AND (m.min_price, m.max_price, m.merchant_count) + IS DISTINCT FROM (s.min_price, s.max_price, s.merchant_count); + SELECT m.item_id INTO v_item FROM rate.master_products m WHERE m.id = p_master_id; + IF v_item IS NOT NULL THEN + PERFORM rate.refresh_item_stats(v_item); + END IF; +END $$; + +-- M2.2 Item fuzzy match (design 4.5 step 5) ----------------------------------------------- + +-- Item names carry no storage, so a bare title never scores against a sized name (#1105). +-- p_brand filters; p_model is accepted for the matcher's call shape but does not filter: +-- an Item's model is the key segment and a different spelling must still be found. +CREATE OR REPLACE FUNCTION rate.match_item( + search_title text, + p_brand text DEFAULT NULL, + p_model text DEFAULT NULL, + p_threshold real DEFAULT 0.4 +) +RETURNS TABLE(item_id uuid, item_key text, name text, similarity real) +LANGUAGE plpgsql +STABLE +SET search_path = '' +AS $$ +DECLARE + v_trgm text; +BEGIN + SELECT n.nspname INTO v_trgm + FROM pg_catalog.pg_extension e JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'pg_trgm'; + IF v_trgm IS NULL THEN + RAISE EXCEPTION 'rate.match_item: pg_trgm extension is not installed'; + END IF; + RETURN QUERY EXECUTE format( + 'SELECT i.id, i.item_key, i.name, %I.similarity(i.name, $1) AS similarity + FROM rate.items i + WHERE i.status <> ''archived'' + AND ($2 IS NULL OR lower(i.brand) = lower($2)) + AND %I.similarity(i.name, $1) >= $3 + ORDER BY 4 DESC, i.id + LIMIT 5', + v_trgm, v_trgm + ) USING search_title, p_brand, p_threshold; +END $$; + +-- M2.3 retire an Item's unknown variant (design 4.5 step 4, M3 rule for one Item) --------- + +-- When a multi-size Item (a live sized variant, a category with a storage axis) still has a +-- live `unknown` variant, its listings move to "no variant" and the variant is archived, in +-- one transaction. Lock order as everywhere else: listings, then the variant, then the Item +-- (through the refreshes), so it cannot deadlock with an ingest moving one of those +-- listings. What it changes is recorded in `metadata` (listing: the variant and source it +-- had; variant: its status before), so the rollback can put it back. Returns the number of +-- listings moved. Price alerts stay on the archived variant (an alert needs a size). +CREATE OR REPLACE FUNCTION rate.retire_item_unknown_variant(p_item_id uuid) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_unknown uuid; + v_status text; + v_moved integer := 0; + v_swept integer := 0; + v_override text := COALESCE(current_setting('rate.variant_override', true), ''); +BEGIN + IF EXISTS (SELECT 1 FROM rate.items i JOIN rate.categories c ON c.id = i.category_id + WHERE i.id = p_item_id AND c.slug = 'earbuds') THEN + RETURN 0; + END IF; + SELECT mp.id INTO v_unknown + FROM rate.master_products mp + WHERE mp.item_id = p_item_id AND mp.variant_key = 'unknown' AND mp.status IS DISTINCT FROM 'archived'; + IF v_unknown IS NULL THEN + -- Already retired: move any listing a racing writer put back on it. + SELECT mp.id INTO v_unknown + FROM rate.master_products mp + WHERE mp.item_id = p_item_id AND mp.variant_key = 'unknown' AND mp.status = 'archived' + AND mp.metadata ->> 'retired_by' = 'retire_item_unknown_variant'; + IF v_unknown IS NULL THEN + RETURN 0; + END IF; + PERFORM set_config('rate.variant_override', 'on', true); + UPDATE rate.products p + SET cluster_id = NULL, + variant_source = NULL, + metadata = COALESCE(p.metadata, '{}'::jsonb) + || jsonb_build_object('retired_unknown_variant', v_unknown, + 'variant_source_before_retire', p.variant_source) + WHERE p.cluster_id = v_unknown; + GET DIAGNOSTICS v_moved = ROW_COUNT; + PERFORM set_config('rate.variant_override', v_override, true); + IF v_moved > 0 THEN + PERFORM rate.refresh_master_market_stats(v_unknown); + PERFORM rate.refresh_item_stats(p_item_id); + END IF; + RETURN v_moved; + END IF; + PERFORM 1 FROM rate.products p WHERE p.cluster_id = v_unknown ORDER BY p.id FOR UPDATE; + SELECT mp.status INTO v_status FROM rate.master_products mp WHERE mp.id = v_unknown FOR UPDATE; + -- Re-check under the locks: a concurrent call may have retired it, or the sized variant + -- may have been archived meanwhile. + IF NOT FOUND OR v_status IS NOT DISTINCT FROM 'archived' OR NOT EXISTS ( + SELECT 1 FROM rate.master_products mp + WHERE mp.item_id = p_item_id AND mp.status IS DISTINCT FROM 'archived' + AND mp.variant_key ~ '^[0-9]+(gb|tb)$') THEN + RETURN 0; + END IF; + PERFORM set_config('rate.variant_override', 'on', true); + UPDATE rate.products p + SET cluster_id = NULL, + variant_source = NULL, + metadata = COALESCE(p.metadata, '{}'::jsonb) + || jsonb_build_object('retired_unknown_variant', v_unknown, + 'variant_source_before_retire', p.variant_source) + WHERE p.cluster_id = v_unknown; + GET DIAGNOSTICS v_moved = ROW_COUNT; + PERFORM set_config('rate.variant_override', v_override, true); + UPDATE rate.master_products mp + SET status = 'archived', + metadata = COALESCE(mp.metadata, '{}'::jsonb) + || jsonb_build_object('status_before_retire', v_status, 'retired_by', 'retire_item_unknown_variant') + WHERE mp.id = v_unknown; + -- Sweep again: a listing written onto the variant while this ran (its writer read the + -- variant before the archive) is moved too; later writes see it archived. + PERFORM set_config('rate.variant_override', 'on', true); + UPDATE rate.products p + SET cluster_id = NULL, + variant_source = NULL, + metadata = COALESCE(p.metadata, '{}'::jsonb) + || jsonb_build_object('retired_unknown_variant', v_unknown, + 'variant_source_before_retire', p.variant_source) + WHERE p.cluster_id = v_unknown; + GET DIAGNOSTICS v_swept = ROW_COUNT; + PERFORM set_config('rate.variant_override', v_override, true); + v_moved := v_moved + v_swept; + PERFORM rate.refresh_master_market_stats(v_unknown); + PERFORM rate.refresh_item_stats(p_item_id); + RETURN v_moved; +END $$; + +-- M2.4 function grants --------------------------------------------------------------------- + +REVOKE EXECUTE ON FUNCTION rate.refresh_item_rating(uuid) FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.refresh_item_stats(uuid) FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.match_item(text, text, text, real) FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.retire_item_unknown_variant(uuid) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.refresh_item_rating(uuid) TO service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.refresh_item_stats(uuid) TO service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.match_item(text, text, text, real) TO service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.retire_item_unknown_variant(uuid) TO service_role, rate_worker; + +-- M1.6 triggers (after the functions they call exist) ------------------------------------- + +DROP TRIGGER IF EXISTS trg_master_products_sync_item ON rate.master_products; +CREATE TRIGGER trg_master_products_sync_item + BEFORE INSERT OR UPDATE OF model_key, status, item_id ON rate.master_products + FOR EACH ROW EXECUTE FUNCTION rate.master_products_sync_item(); +DROP TRIGGER IF EXISTS trg_master_products_move_listings ON rate.master_products; +CREATE TRIGGER trg_master_products_move_listings + AFTER UPDATE OF model_key, item_id ON rate.master_products + FOR EACH ROW EXECUTE FUNCTION rate.master_products_move_listings(); +DROP TRIGGER IF EXISTS trg_products_sync_item ON rate.products; +CREATE TRIGGER trg_products_sync_item + BEFORE INSERT OR UPDATE OF cluster_id, item_id, variant_source ON rate.products + FOR EACH ROW EXECUTE FUNCTION rate.products_sync_item(); +DROP TRIGGER IF EXISTS trg_products_refresh_left ON rate.products; +CREATE TRIGGER trg_products_refresh_left + AFTER UPDATE OF cluster_id, item_id ON rate.products + FOR EACH ROW EXECUTE FUNCTION rate.products_refresh_left(); +DROP TRIGGER IF EXISTS trg_products_recheck_retired_variant ON rate.products; +CREATE CONSTRAINT TRIGGER trg_products_recheck_retired_variant + AFTER INSERT OR UPDATE OF cluster_id ON rate.products + DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW EXECUTE FUNCTION rate.products_recheck_retired_variant(); + +-- M2.5 backfill (design 7.1) --------------------------------------------------------------- + +-- Abort before writing if two Items would share a slug (the M0 inventory found none). +DO $$ +DECLARE + v_clash text; +BEGIN + SELECT string_agg(slug, ', ') INTO v_clash FROM ( + SELECT rate.item_slug_from_master(mp.slug, mp.model_key) AS slug + FROM rate.master_products mp + WHERE mp.item_id IS NULL + GROUP BY 1 + HAVING count(DISTINCT split_part(mp.model_key, ':', 1) || ':' || split_part(mp.model_key, ':', 2)) > 1 + ) c; + IF v_clash IS NOT NULL THEN + RAISE EXCEPTION 'M2 backfill: Item slug collisions, resolve them first: %', v_clash; + END IF; + -- ...or take the slug of an Item that already exists under another key. + SELECT string_agg(DISTINCT i.slug, ', ') INTO v_clash + FROM rate.master_products mp + JOIN rate.items i ON i.slug = rate.item_slug_from_master(mp.slug, mp.model_key) + WHERE mp.item_id IS NULL + AND i.item_key <> split_part(mp.model_key, ':', 1) || ':' || split_part(mp.model_key, ':', 2); + IF v_clash IS NOT NULL THEN + RAISE EXCEPTION 'M2 backfill: Item slugs already taken by other Items: %', v_clash; + END IF; +END $$; + +UPDATE rate.master_products mp +SET variant_key = NULLIF(split_part(mp.model_key, ':', 3), '') +WHERE mp.variant_key IS DISTINCT FROM NULLIF(split_part(mp.model_key, ':', 3), ''); + +-- One Item per brand:model, including Items whose masters are all archived. Representative +-- variant: published first, then most merchants, then oldest. Item specs carry no storage or +-- RAM (variant overrides, design 4.1). +INSERT INTO rate.items (item_key, slug, name, brand, model, category_id, status, entity_type, + description, specifications, key_specs, spec_provenance, images, thumbnail, + expert_rating, expert_review_summary, expert_criteria_scores, metadata) +SELECT DISTINCT ON (k.item_key) + k.item_key, + rate.item_slug_from_master(mp.slug, mp.model_key), + rate.item_name_from_master(mp.name), + mp.brand, mp.model, mp.category_id, + CASE WHEN bool_or(mp.status = 'published') OVER w THEN 'published' + WHEN bool_or(COALESCE(mp.status, 'draft') <> 'archived') OVER w THEN 'draft' + ELSE 'archived' END, + COALESCE(mp.entity_type, 'product'), mp.description, + COALESCE(mp.specifications, '{}'::jsonb) #- '{hardware,storage_gb}' #- '{hardware,ram_gb}', + COALESCE(mp.key_specs, '{}'::jsonb) - 'storage' - 'ram', + COALESCE(mp.spec_provenance, '{}'::jsonb), COALESCE(mp.images, '{}'), mp.thumbnail, + mp.expert_rating, mp.expert_review_summary, COALESCE(mp.expert_criteria_scores, '{}'::jsonb), + COALESCE(mp.metadata, '{}'::jsonb) || jsonb_build_object('backfilled_from_master', mp.id) +FROM rate.master_products mp +CROSS JOIN LATERAL ( + SELECT split_part(mp.model_key, ':', 1) || ':' || split_part(mp.model_key, ':', 2) AS item_key +) k +WINDOW w AS (PARTITION BY k.item_key) +ORDER BY k.item_key, (mp.status = 'published') DESC NULLS LAST, + mp.merchant_count DESC NULLS LAST, mp.created_at ASC NULLS LAST, mp.id +ON CONFLICT (item_key) DO NOTHING; + +UPDATE rate.master_products mp SET item_id = i.id +FROM rate.items i +WHERE mp.item_id IS DISTINCT FROM i.id + AND i.item_key = split_part(mp.model_key, ':', 1) || ':' || split_part(mp.model_key, ':', 2); + +-- Re-runnable and self-correcting: also fixes listings whose master changed after a run. +UPDATE rate.products p +SET item_id = mp.item_id, variant_source = COALESCE(p.variant_source, 'legacy') +FROM rate.master_products mp +WHERE p.cluster_id = mp.id + AND (p.item_id IS DISTINCT FROM mp.item_id OR p.variant_source IS NULL); + +-- Items that already existed (created by the sync trigger, or by a re-run) take their status +-- from their masters, so a published product is never hidden behind a draft Item. +UPDATE rate.items i SET status = s.status +FROM ( + SELECT item_id, + CASE WHEN bool_or(status = 'published') THEN 'published' + WHEN bool_or(COALESCE(status, 'draft') <> 'archived') THEN 'draft' + ELSE 'archived' END AS status + FROM rate.master_products WHERE item_id IS NOT NULL GROUP BY item_id +) s +WHERE i.id = s.item_id AND i.status IS DISTINCT FROM s.status; + +-- Item aggregates and ratings. +DO $$ +DECLARE + v_id uuid; +BEGIN + FOR v_id IN SELECT i.id FROM rate.items i ORDER BY i.id LOOP + PERFORM rate.refresh_item_stats(v_id); + END LOOP; +END $$; + +COMMIT; diff --git a/Docs/Proposals/2026-10-29-dos-me-item-rpcs-rollback.sql b/Docs/Proposals/2026-10-29-dos-me-item-rpcs-rollback.sql new file mode 100644 index 000000000..7292a92f5 --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-item-rpcs-rollback.sql @@ -0,0 +1,11 @@ +-- PROPOSAL, NOT APPLIED. Rollback of 2026-10-29-dos-me-item-rpcs.sql (DOS/Rate#1160). New +-- functions only, no data: dropping them is enough. Turn the web's Item catalogue off first +-- (RATE_ITEM_MODEL off) or it falls back to the variant RPCs. Re-runnable. + +BEGIN; +DROP FUNCTION IF EXISTS rate.search_catalog_autocomplete_v2(TEXT, INTEGER); +DROP FUNCTION IF EXISTS rate.get_category_brand_facets_v2(UUID); +DROP FUNCTION IF EXISTS rate.search_items_v1( + UUID, TEXT[], TEXT, NUMERIC, NUMERIC, NUMERIC, INTEGER[], BOOLEAN, TEXT, INTEGER, INTEGER +); +COMMIT; diff --git a/Docs/Proposals/2026-10-29-dos-me-item-rpcs.sql b/Docs/Proposals/2026-10-29-dos-me-item-rpcs.sql new file mode 100644 index 000000000..f851baeab --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-item-rpcs.sql @@ -0,0 +1,265 @@ +-- PROPOSAL, NOT APPLIED. DOS-Me migration: Item catalogue RPCs (DOS/Rate#1160, #1275; design +-- Docs/Proposals/2026-10-27-item-variant-listing-model.md section 5.1). Applies any time AFTER +-- 2026-10-28-dos-me-m1-m2-items.sql (needs rate.items and master_products.item_id); it does not +-- wait for M3/M4. Rollback: 2026-10-29-dos-me-item-rpcs-rollback.sql. +-- +-- New functions only; the variant RPCs (search_master_products, get_category_brand_facets, +-- search_catalog_autocomplete) are untouched, so the flag-off web keeps working. +-- Idempotent: CREATE OR REPLACE and grants. + +BEGIN; + +-- 1. search_items_v1: one row per published Item, with the published variants the filters +-- match. Same filter names as search_master_products where they apply; `p_storage_gb` means +-- "has a variant of that size" (design 5.1); price filters and sorts use the cheapest priced +-- matching variant (`matching_min_price`). `p_search` matches the Item's and its published +-- variants' names, keys and chips. The card links to the representative variant (the +-- cheapest priced matching one, else the first matching one) and shows `chips`, one per +-- published variant with `matching` set. `total_count` is the number of matching Items (for +-- paging), the same on every row. +CREATE OR REPLACE FUNCTION rate.search_items_v1( + p_category_id UUID DEFAULT NULL, + p_brands TEXT[] DEFAULT NULL, + p_search TEXT DEFAULT NULL, + p_min_price NUMERIC DEFAULT NULL, + p_max_price NUMERIC DEFAULT NULL, + p_min_rating NUMERIC DEFAULT NULL, + p_storage_gb INTEGER[] DEFAULT NULL, + p_has_5g BOOLEAN DEFAULT NULL, + p_sort TEXT DEFAULT 'default', + p_limit INTEGER DEFAULT 24, + p_offset INTEGER DEFAULT 0 +) +RETURNS TABLE ( + item_id UUID, + item_key TEXT, + slug TEXT, + name TEXT, + brand TEXT, + model TEXT, + category_id UUID, + thumbnail TEXT, + min_price NUMERIC, + max_price NUMERIC, + matching_min_price NUMERIC, + matching_max_price NUMERIC, + price_variant_count INTEGER, + merchant_count INTEGER, + listing_count INTEGER, + user_rating_weighted NUMERIC, + authentic_review_count INTEGER, + expert_rating NUMERIC, + average_rating NUMERIC, + total_reviews INTEGER, + default_variant_id UUID, + representative_variant_id UUID, + representative_variant_slug TEXT, + variant_keys TEXT[], + matching_variant_keys TEXT[], + chips JSONB, + created_at TIMESTAMPTZ, + total_count BIGINT +) +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = '' +AS $$ + WITH v AS ( + SELECT m.id, m.slug, m.item_id, m.variant_key, m.min_price, m.max_price, m.storage_gb, m.sort_order, + COALESCE( + (p_storage_gb IS NULL OR cardinality(p_storage_gb) = 0 OR m.storage_gb = ANY (p_storage_gb)) + AND (p_has_5g IS NULL OR p_has_5g = false OR m.has_5g = true), + false) AS matching, + pg_catalog.concat_ws(' ', m.name, m.model_key, COALESCE(m.key_specs ->> 'chip', '')) AS search_text + FROM rate.master_products m + WHERE m.status = 'published' AND m.item_id IS NOT NULL + ), + per_item AS ( + SELECT + v.item_id, + pg_catalog.array_agg(v.variant_key ORDER BY v.sort_order NULLS LAST, v.storage_gb NULLS LAST, v.variant_key) + AS variant_keys, + pg_catalog.array_agg(v.variant_key ORDER BY v.sort_order NULLS LAST, v.storage_gb NULLS LAST, v.variant_key) + FILTER (WHERE v.matching) AS matching_variant_keys, + min(v.min_price) FILTER (WHERE v.matching AND v.min_price > 0) AS matching_min_price, + max(GREATEST(v.max_price, v.min_price)) FILTER (WHERE v.matching AND v.min_price > 0) AS matching_max_price, + (pg_catalog.array_agg(v.id ORDER BY (v.min_price > 0) DESC NULLS LAST, v.min_price NULLS LAST, + v.sort_order NULLS LAST, v.storage_gb NULLS LAST, v.variant_key) FILTER (WHERE v.matching))[1] + AS representative_variant_id, + pg_catalog.jsonb_agg(pg_catalog.jsonb_build_object( + 'variant_id', v.id, 'variant_key', v.variant_key, 'slug', v.slug, + 'storage_gb', v.storage_gb, 'min_price', NULLIF(v.min_price, 0), 'matching', v.matching) + ORDER BY v.sort_order NULLS LAST, v.storage_gb NULLS LAST, v.variant_key) AS chips, + pg_catalog.string_agg(v.search_text, ' ') AS variant_search_text + FROM v + GROUP BY v.item_id + ), + matched AS ( + SELECT i.*, p.variant_keys, p.matching_variant_keys, p.matching_min_price, p.matching_max_price, + p.representative_variant_id, p.chips + FROM rate.items i + JOIN per_item p ON p.item_id = i.id + WHERE i.status = 'published' + AND p.matching_variant_keys IS NOT NULL + AND (p_category_id IS NULL OR i.category_id = p_category_id) + AND ( + p_brands IS NULL + OR cardinality(p_brands) = 0 + OR pg_catalog.lower(pg_catalog.btrim(i.brand)) = ANY ( + SELECT pg_catalog.lower(pg_catalog.btrim(b)) FROM unnest(p_brands) AS b + ) + ) + AND ( + p_search IS NULL + OR pg_catalog.btrim(p_search) = '' + OR pg_catalog.concat_ws(' ', i.brand, i.name, i.model, i.item_key, + COALESCE(i.key_specs ->> 'chip', ''), + COALESCE(i.specifications #>> '{hardware,chipset}', ''), + p.variant_search_text) + ILIKE '%' || pg_catalog.regexp_replace(pg_catalog.btrim(p_search), '([%_\\])', '\\\1', 'g') || '%' ESCAPE '\' + ) + AND (p_min_price IS NULL OR p_min_price <= 0 OR p.matching_min_price >= p_min_price) + AND (p_max_price IS NULL OR p_max_price <= 0 OR p.matching_min_price <= p_max_price) + AND ( + p_min_rating IS NULL + OR p_min_rating <= 0 + OR COALESCE(NULLIF(i.user_rating_weighted, 0), NULLIF(i.expert_rating, 0), i.average_rating, 0) + >= p_min_rating + ) + ) + SELECT + m.id, m.item_key, m.slug, m.name, m.brand, m.model, m.category_id, + COALESCE(m.thumbnail, m.images[1]), + m.min_price, m.max_price, m.matching_min_price, m.matching_max_price, m.price_variant_count, + m.merchant_count, m.listing_count, m.user_rating_weighted, m.authentic_review_count, + m.expert_rating, m.average_rating, m.total_reviews, m.default_variant_id, + m.representative_variant_id, rv.slug, + m.variant_keys, m.matching_variant_keys, m.chips, m.created_at, + pg_catalog.count(*) OVER () + FROM matched m + LEFT JOIN rate.master_products rv ON rv.id = m.representative_variant_id + ORDER BY + CASE WHEN p_sort = 'price_asc' THEN m.matching_min_price END ASC NULLS LAST, + CASE WHEN p_sort = 'price_desc' THEN m.matching_min_price END DESC NULLS LAST, + CASE WHEN p_sort = 'merchants_desc' THEN m.merchant_count END DESC NULLS LAST, + CASE WHEN p_sort = 'rating_desc' THEN COALESCE(NULLIF(m.user_rating_weighted, 0), m.average_rating, 0) END + DESC NULLS LAST, + m.created_at DESC, + m.id ASC + LIMIT GREATEST(1, LEAST(COALESCE(p_limit, 24), 200)) + OFFSET GREATEST(0, COALESCE(p_offset, 0)); +$$; + +-- 2. Brand facets counted by published Item (design 5.1). +CREATE OR REPLACE FUNCTION rate.get_category_brand_facets_v2(p_category_id UUID) +RETURNS TABLE (brand TEXT, count INTEGER) +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = '' +AS $$ + SELECT i.brand, COUNT(*)::INTEGER AS count + FROM rate.items i + WHERE i.status = 'published' + AND i.category_id = p_category_id + AND i.brand IS NOT NULL + AND pg_catalog.btrim(i.brand) <> '' + AND EXISTS (SELECT 1 FROM rate.master_products m + WHERE m.item_id = i.id AND m.status = 'published') + GROUP BY i.brand + ORDER BY count DESC, i.brand ASC; +$$; + +-- 3. Autocomplete by Item. SECURITY DEFINER like search_catalog_autocomplete: anon has no +-- USAGE on the pg_trgm schema; the body reads published Items only. Same result shape as v1 +-- with entity_kind 'item'. +CREATE OR REPLACE FUNCTION rate.search_catalog_autocomplete_v2( + p_query TEXT, + p_limit INTEGER DEFAULT 8 +) +RETURNS TABLE ( + entity_kind TEXT, + id UUID, + slug TEXT, + title TEXT, + subtitle TEXT, + brand_or_platform TEXT, + thumbnail TEXT, + min_price NUMERIC, + max_price NUMERIC, + rating NUMERIC, + review_count INTEGER, + merchant_or_official_count INTEGER, + similarity_score REAL +) +LANGUAGE plpgsql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_trgm_schema TEXT; + v_clean_q TEXT := pg_catalog.left(pg_catalog.btrim(COALESCE(p_query, '')), 100); + v_escaped_q TEXT; + v_limit INTEGER := LEAST(GREATEST(COALESCE(p_limit, 8), 1), 25); +BEGIN + IF v_clean_q = '' THEN + RETURN; + END IF; + v_escaped_q := pg_catalog.replace(pg_catalog.replace(pg_catalog.replace( + v_clean_q, '\', '\\'), '%', '\%'), '_', '\_'); + SELECT n.nspname INTO v_trgm_schema + FROM pg_catalog.pg_extension e JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'pg_trgm'; + IF v_trgm_schema IS NULL THEN + RAISE EXCEPTION 'rate.search_catalog_autocomplete_v2: pg_trgm extension is not installed'; + END IF; + RETURN QUERY EXECUTE pg_catalog.format( + 'SELECT * FROM ( + SELECT + ''item''::TEXT, + i.id, + i.slug, + i.name, + COALESCE(i.key_specs ->> ''chip'', i.model), + i.brand, + COALESCE(i.thumbnail, i.images[1]), + i.min_price, + i.max_price, + COALESCE(NULLIF(i.user_rating_weighted, 0), i.average_rating, 0)::NUMERIC, + COALESCE(NULLIF(i.authentic_review_count, 0), i.total_reviews, 0)::INTEGER, + COALESCE(i.merchant_count, 0)::INTEGER, + GREATEST( + %I.similarity(pg_catalog.lower(i.name), pg_catalog.lower($1)), + %I.similarity(pg_catalog.lower(i.brand || '' '' || i.model), pg_catalog.lower($1)), + CASE WHEN pg_catalog.lower(i.name) LIKE ''%%'' || pg_catalog.lower($2) || ''%%'' ESCAPE ''\'' + THEN 0.75::real ELSE 0.0::real END + ) AS score + FROM rate.items i + WHERE i.status = ''published'' + AND EXISTS (SELECT 1 FROM rate.master_products m + WHERE m.item_id = i.id AND m.status = ''published'') + ) s + WHERE s.score >= 0.2 + ORDER BY s.score DESC, s.id + LIMIT $3', + v_trgm_schema, v_trgm_schema + ) USING v_clean_q, v_escaped_q, v_limit; +END $$; + +-- 4. Grants: the web reads as anon / authenticated; the worker and service role too. +REVOKE EXECUTE ON FUNCTION rate.search_items_v1( + UUID, TEXT[], TEXT, NUMERIC, NUMERIC, NUMERIC, INTEGER[], BOOLEAN, TEXT, INTEGER, INTEGER +) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION rate.get_category_brand_facets_v2(UUID) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION rate.search_catalog_autocomplete_v2(TEXT, INTEGER) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION rate.search_items_v1( + UUID, TEXT[], TEXT, NUMERIC, NUMERIC, NUMERIC, INTEGER[], BOOLEAN, TEXT, INTEGER, INTEGER +) TO anon, authenticated, service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.get_category_brand_facets_v2(UUID) + TO anon, authenticated, service_role, rate_worker; +GRANT EXECUTE ON FUNCTION rate.search_catalog_autocomplete_v2(TEXT, INTEGER) + TO anon, authenticated, service_role, rate_worker; + +COMMIT; diff --git a/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-rollback.sql b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-rollback.sql new file mode 100644 index 000000000..33563a79a --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-rollback.sql @@ -0,0 +1,39 @@ +-- PROPOSAL, NOT APPLIED. Rollback of 2026-10-29-dos-me-m0-seed-rekey.sql (DOS/Rate#1212). +-- Run only BEFORE M1+M2 (once Items exist, roll M1+M2 back first: Items are keyed on the new +-- model_keys). Revert the seed change in the same step. Re-runnable: it only touches rows +-- that still carry the M0 markers in metadata. + +BEGIN; + +-- 1. Restore the previous keys. A key taken again meanwhile (by a newly seeded master) is +-- skipped with a NOTICE and left for a manual decision. +DO $$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT id, model_key, metadata ->> 'm0_rekeyed_from' AS old_key + FROM rate.master_products + WHERE metadata ? 'm0_rekeyed_from' + LOOP + IF EXISTS (SELECT 1 FROM rate.master_products WHERE model_key = r.old_key AND id <> r.id) THEN + RAISE NOTICE 'M0 rollback skip %: % is taken again', r.model_key, r.old_key; + ELSE + UPDATE rate.master_products + SET model_key = r.old_key, + metadata = metadata - 'm0_rekeyed_from', + updated_at = now() + WHERE id = r.id; + END IF; + END LOOP; +END +$$; + +-- 2. Restore the earbuds storage values. +UPDATE rate.master_products + SET storage_gb = (metadata ->> 'm0_storage_gb_before')::numeric, + metadata = metadata - 'm0_storage_gb_before', + updated_at = now() + WHERE metadata ? 'm0_storage_gb_before'; + +COMMIT; diff --git a/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-verify.sql b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-verify.sql new file mode 100644 index 000000000..ec97d9047 --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey-verify.sql @@ -0,0 +1,39 @@ +-- PROPOSAL. Read-only checks for 2026-10-29-dos-me-m0-seed-rekey.sql (DOS/Rate#1212). Run +-- before (as the pre-check) and after. `scripts/test-m0-seed-rekey-proposal.mjs` runs the +-- expect-0 ones on the replay stack. + +-- V1 No master still on a drifted seed key. expect 0 +SELECT count(*) AS drifted_keys +FROM rate.master_products +WHERE model_key IN ('samsung:galaxy-buds3-pro:64gb', 'apple:airpods-pro-3:standard', + 'sony:wf-1000xm5:64gb', 'sony:wf-1000xm6:standard', + 'apple:airpods-pro-2-usbc:64gb', 'apple:watch-ultra-3-49mm:64gb'); + +-- V2 Earbuds carry no storage size, in the key or the column, category or not. expect 0 +SELECT count(*) AS sized_earbuds +FROM rate.master_products mp LEFT JOIN rate.categories c ON c.id = mp.category_id +WHERE (c.slug = 'earbuds' OR mp.model_key IN ('samsung:galaxy-buds3-pro:unknown', 'apple:airpods-pro-3:unknown', + 'sony:wf-1000xm5:unknown', 'sony:wf-1000xm6:unknown', + 'apple:airpods-pro-2:unknown')) + AND (split_part(mp.model_key, ':', 3) <> 'unknown' OR mp.storage_gb IS NOT NULL); + +-- V3 No two masters share a model_key (also after a re-key). expect 0 +SELECT count(*) AS duplicate_keys +FROM (SELECT model_key FROM rate.master_products GROUP BY 1 HAVING count(*) > 1) d; + +-- V4 Every listing still points at an existing master. expect 0 +SELECT count(*) AS orphan_listings +FROM rate.products p +WHERE p.cluster_id IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM rate.master_products mp WHERE mp.id = p.cluster_id); + +-- V5 Pre-check (information): every key this step touches, old and new, with its row. +SELECT model_key, id, status, storage_gb, slug +FROM rate.master_products +WHERE model_key IN ('samsung:galaxy-buds3-pro:64gb', 'samsung:galaxy-buds3-pro:unknown', + 'apple:airpods-pro-3:standard', 'apple:airpods-pro-3:unknown', + 'sony:wf-1000xm5:64gb', 'sony:wf-1000xm5:unknown', + 'sony:wf-1000xm6:standard', 'sony:wf-1000xm6:unknown', + 'apple:airpods-pro-2-usbc:64gb', 'apple:airpods-pro-2:unknown', + 'apple:watch-ultra-3-49mm:64gb', 'apple:watch-ultra-3:64gb') +ORDER BY model_key; diff --git a/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey.md b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey.md new file mode 100644 index 000000000..83463531f --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey.md @@ -0,0 +1,56 @@ +# Proposal: DOS-Me data step M0, seed key re-key (DOS/Rate#1212, #1123) + +**Status:** proposal for JOY, **not applied**. A production data write: JOY approves and applies it through DOS-Me (preview, then beta, then prod), **before** the M1+M2 Item migration ([`2026-10-28-dos-me-m1-m2-items.md`](./2026-10-28-dos-me-m1-m2-items.md)). + +Inventory: the M0 comment on #1150 (2026-10-28, anon read of prod: 37 masters, 6 drifted keys, no duplicate, no slug collision). Design: [`2026-10-27-item-variant-listing-model.md`](./2026-10-27-item-variant-listing-model.md), section 7 "M0". + +## Files + +| File | What | +| :----------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`2026-10-29-dos-me-m0-seed-rekey.sql`](./2026-10-29-dos-me-m0-seed-rekey.sql) | Data only, one transaction, idempotent. | +| [`2026-10-29-dos-me-m0-seed-rekey-rollback.sql`](./2026-10-29-dos-me-m0-seed-rekey-rollback.sql) | Restores keys and storage from the recorded markers; re-runnable. Only before M1+M2 (or after rolling M1+M2 back). | +| [`2026-10-29-dos-me-m0-seed-rekey-verify.sql`](./2026-10-29-dos-me-m0-seed-rekey-verify.sql) | Read-only: V1 to V4 expect 0 after; V5 is the pre-check listing every touched key. | +| [`scripts/test-m0-seed-rekey-proposal.mjs`](../../scripts/test-m0-seed-rekey-proposal.mjs) | CI replay: the six drifted masters with listings, M0 twice, its checks, M1+M2 on top (all Item checks pass, Items under the normalizer keys), both rollbacks, and the skip when a target key is taken. | + +## What it changes + +| Stored `model_key` | New `model_key` | Status (inventory) | Listings | +| :------------------------------ | :--------------------------------- | :----------------- | -------: | +| `samsung:galaxy-buds3-pro:64gb` | `samsung:galaxy-buds3-pro:unknown` | published | 4 | +| `apple:airpods-pro-3:standard` | `apple:airpods-pro-3:unknown` | published | 9 | +| `sony:wf-1000xm5:64gb` | `sony:wf-1000xm5:unknown` | draft | 8 | +| `sony:wf-1000xm6:standard` | `sony:wf-1000xm6:unknown` | draft | 9 | +| `apple:airpods-pro-2-usbc:64gb` | `apple:airpods-pro-2:unknown` | draft | 4 | +| `apple:watch-ultra-3-49mm:64gb` | `apple:watch-ultra-3:64gb` | draft | 4 | + +- **Re-key in place.** Ids and slugs do not change, so listings, price alerts, reviews and public URLs keep pointing at the same rows; nothing is archived or merged. A rename whose target key is already taken is skipped with a NOTICE and shows in V1 (a reviewed merge, never an automatic one; the inventory found none). +- **Earbuds carry no storage:** `storage_gb = NULL` on every earbuds master (today 64, 64, 64, 1, 1 on the five rows above). +- **Markers for the rollback:** `metadata.m0_rekeyed_from` (old key) and `metadata.m0_storage_gb_before`. +- `apple:airpods-pro-2-usbc` becomes Item `apple:airpods-pro-2`: the normalizer does not split USB-C from Lightning (only USB-C is sold new today). + +## Order of operations + +1. Pre-check: run V5 (and V1 to V4) on beta and prod; compare with the table above. +2. Apply this file (beta, verify, then prod, verify). +3. **Right after**, merge the seed change below. Do not run `jobs/backfill-specs.ts --apply` between steps 2 and 3: it looks masters up by the seed key and inserts one when none is found, which would re-create the old keys as duplicates. +4. Then M1+M2 (`2026-10-28-dos-me-m1-m2-items.sql`). + +## Seed change that ships right after (not in this PR) + +`apps/importer/src/shopee/flagship-seed-data.ts`: + +| Seed | `model_key` | `storage_gb` | +| :----------------------- | :-------------------------------------------------------------------- | :----------- | +| AirPods Pro 2 USB-C | `apple:airpods-pro-2-usbc:64gb` -> `apple:airpods-pro-2:unknown` | 64 -> `null` | +| Sony WF-1000XM5 | `sony:wf-1000xm5:64gb` -> `sony:wf-1000xm5:unknown` | 64 -> `null` | +| Samsung Galaxy Buds3 Pro | `samsung:galaxy-buds3-pro:64gb` -> `samsung:galaxy-buds3-pro:unknown` | 64 -> `null` | +| Apple Watch Ultra 3 | `apple:watch-ultra-3-49mm:64gb` -> `apple:watch-ultra-3:64gb` | unchanged | +| AirPods Pro 3 USB-C | `apple:airpods-pro-3:standard` -> `apple:airpods-pro-3:unknown` | -> `null` | +| Sony WF-1000XM6 | `sony:wf-1000xm6:standard` -> `sony:wf-1000xm6:unknown` | -> `null` | + +(The `FlagshipDeviceSeed.storage_gb` type is `number | null`, so `null` needs no type change.) The Core session opens this as a separate PR once JOY schedules the data step; it must not merge before the data is re-keyed. + +## Rollback + +`2026-10-29-dos-me-m0-seed-rekey-rollback.sql`, with the seed change reverted in the same step. If M1+M2 is applied, roll it back first (its Items are keyed on the new keys). A key that a newly seeded master took meanwhile is skipped with a NOTICE. diff --git a/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey.sql b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey.sql new file mode 100644 index 000000000..404e87d77 --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m0-seed-rekey.sql @@ -0,0 +1,77 @@ +-- PROPOSAL, NOT APPLIED. DOS-Me data step M0 for the Item model (DOS/Rate#1212, #1123; +-- design Docs/Proposals/2026-10-27-item-variant-listing-model.md section 7 "M0"; inventory in +-- the #1150 comment of 2026-10-28). JOY reviews and applies it through DOS-Me (preview, beta, +-- prod) BEFORE 2026-10-28-dos-me-m1-m2-items.sql. Rollback: +-- 2026-10-29-dos-me-m0-seed-rekey-rollback.sql. Verification: +-- 2026-10-29-dos-me-m0-seed-rekey-verify.sql. +-- +-- What it does (data only, no DDL): +-- 1. Re-keys the six flagship-seed masters whose stored model_key the normalizer never +-- produces, so new listings hit Tier 1 and M2 creates their Items under the normalizer's +-- Item keys. Ids and slugs do not change, so listings, alerts, reviews and public URLs +-- keep pointing at the same rows. A rename whose target key is already taken by another +-- master is skipped with a NOTICE (that pair needs a reviewed merge, not a rename; the +-- inventory found none). Duplicates are never deleted. +-- 2. Clears storage_gb on earbuds masters (earbuds have no storage axis, #643/#657). +-- Each change records the previous value in metadata (m0_rekeyed_from, m0_storage_gb_before) +-- so the rollback can restore it. +-- +-- The matching seed change (flagship-seed-data.ts keys, earbuds storage_gb: null) must merge +-- right AFTER this is applied, and `jobs/backfill-specs.ts --apply` must not run in between: +-- it looks masters up by the seed key and inserts one when none is found. +-- +-- Idempotent: a master already on its new key is not touched; the storage step only touches +-- rows that still hold a storage. + +BEGIN; + +-- 1. Re-key ---------------------------------------------------------------------------------- + +DO $$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT * FROM (VALUES + ('samsung:galaxy-buds3-pro:64gb', 'samsung:galaxy-buds3-pro:unknown'), + ('apple:airpods-pro-3:standard', 'apple:airpods-pro-3:unknown'), + ('sony:wf-1000xm5:64gb', 'sony:wf-1000xm5:unknown'), + ('sony:wf-1000xm6:standard', 'sony:wf-1000xm6:unknown'), + -- The normalizer does not split USB-C from Lightning: one Item, apple:airpods-pro-2. + ('apple:airpods-pro-2-usbc:64gb', 'apple:airpods-pro-2:unknown'), + -- The Watch Ultra comes in one case size, so its key carries none. + ('apple:watch-ultra-3-49mm:64gb', 'apple:watch-ultra-3:64gb') + ) AS t(old_key, new_key) + LOOP + IF NOT EXISTS (SELECT 1 FROM rate.master_products WHERE model_key = r.old_key) THEN + CONTINUE; + ELSIF EXISTS (SELECT 1 FROM rate.master_products WHERE model_key = r.new_key) THEN + RAISE NOTICE 'M0 skip %: % already exists, merge manually', r.old_key, r.new_key; + ELSE + UPDATE rate.master_products + SET model_key = r.new_key, + metadata = COALESCE(metadata, '{}'::jsonb) || jsonb_build_object('m0_rekeyed_from', r.old_key), + updated_at = now() + WHERE model_key = r.old_key; + END IF; + END LOOP; +END +$$; + +-- 2. Earbuds carry no storage ---------------------------------------------------------------- + +-- By category, and by the re-keyed earbuds keys for a master whose category_id is NULL +-- (backfill-specs can insert one without a category). +UPDATE rate.master_products mp + SET metadata = COALESCE(mp.metadata, '{}'::jsonb) + || jsonb_build_object('m0_storage_gb_before', mp.storage_gb), + storage_gb = NULL, + updated_at = now() + WHERE mp.storage_gb IS NOT NULL + AND (mp.model_key IN ('samsung:galaxy-buds3-pro:unknown', 'apple:airpods-pro-3:unknown', + 'sony:wf-1000xm5:unknown', 'sony:wf-1000xm6:unknown', + 'apple:airpods-pro-2:unknown') + OR EXISTS (SELECT 1 FROM rate.categories c + WHERE c.id = mp.category_id AND c.slug = 'earbuds')); + +COMMIT; diff --git a/Docs/Proposals/2026-10-29-dos-me-m3-m4-rollback.sql b/Docs/Proposals/2026-10-29-dos-me-m3-m4-rollback.sql new file mode 100644 index 000000000..b46431b36 --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m3-m4-rollback.sql @@ -0,0 +1,63 @@ +-- PROPOSAL, NOT APPLIED. Rollback of 2026-10-29-dos-me-m3-m4.sql (DOS/Rate#1160). M4 first, +-- then M3 from its backup table. Run with the writers paused. Re-runnable. + +-- M4 --------------------------------------------------------------------------------------- +BEGIN; +DROP TRIGGER IF EXISTS trg_master_products_fill_default_variant ON rate.master_products; +DROP FUNCTION IF EXISTS rate.items_fill_default_variant(); +DROP TRIGGER IF EXISTS trg_products_variant_item_guard ON rate.products; +DROP FUNCTION IF EXISTS rate.products_variant_item_guard(); +ALTER TABLE rate.products DROP CONSTRAINT IF EXISTS products_item_id_not_null; +ALTER TABLE rate.master_products ALTER COLUMN item_id DROP NOT NULL; +-- default_variant_id values stay: nullable data M1+M2 already allowed. +COMMIT; + +-- M3 --------------------------------------------------------------------------------------- +BEGIN; +DO $$ +DECLARE + r record; + v_skipped bigint; +BEGIN + IF to_regclass('rate.m3_unknown_variant_backup') IS NULL THEN + RETURN; + END IF; + -- The variants first (so the listings' triggers see them live), then their listings, with + -- the hand-set-variant guard switched off for this transaction. + PERFORM set_config('rate.variant_override', 'on', true); + FOR r IN + SELECT DISTINCT ON (b.master_id) b.master_id, b.master_status + FROM rate.m3_unknown_variant_backup b + WHERE b.listing_id IS NULL + ORDER BY b.master_id, b.id + LOOP + UPDATE rate.master_products mp + SET status = r.master_status, + metadata = mp.metadata - 'status_before_retire' - 'retired_by' + WHERE mp.id = r.master_id; + END LOOP; + -- Only a listing still carrying the retire marker for that variant: any later variant or + -- Item change cleared the marker (M1 trigger), and that change wins over the rollback. + UPDATE rate.products p + SET cluster_id = b.listing_cluster_id, + variant_source = b.listing_variant_src, + metadata = p.metadata - 'retired_unknown_variant' - 'variant_source_before_retire' + FROM rate.m3_unknown_variant_backup b + WHERE b.listing_id = p.id + AND p.cluster_id IS NULL + AND p.item_id = b.item_id + AND p.metadata ->> 'retired_unknown_variant' = b.master_id::text; + SELECT count(*) INTO v_skipped + FROM rate.m3_unknown_variant_backup b + JOIN rate.products p ON p.id = b.listing_id + WHERE p.cluster_id IS DISTINCT FROM b.listing_cluster_id; + IF v_skipped > 0 THEN + RAISE NOTICE 'M3 rollback: % listing(s) changed after M3 were left as they are', v_skipped; + END IF; + FOR r IN SELECT DISTINCT b.master_id, b.item_id FROM rate.m3_unknown_variant_backup b LOOP + PERFORM rate.refresh_master_market_stats(r.master_id); + PERFORM rate.refresh_item_stats(r.item_id); + END LOOP; +END $$; +DROP TABLE IF EXISTS rate.m3_unknown_variant_backup; +COMMIT; diff --git a/Docs/Proposals/2026-10-29-dos-me-m3-m4-verify.sql b/Docs/Proposals/2026-10-29-dos-me-m3-m4-verify.sql new file mode 100644 index 000000000..d7610950e --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m3-m4-verify.sql @@ -0,0 +1,49 @@ +-- PROPOSAL. Read-only checks for 2026-10-29-dos-me-m3-m4.sql and 2026-10-29-dos-me-item-rpcs.sql +-- (DOS/Rate#1160; design 7.3). Run the M1+M2 verification file too. The CI replay +-- (scripts/test-m3-m4-item-rpcs-proposal.mjs) runs the expect-0 ones. + +-- V1 No listing on a live unknown variant of a multi-size Item (after M3). expect 0 +SELECT count(*) AS unknown_on_multi_size +FROM rate.products p +JOIN rate.master_products mp ON mp.id = p.cluster_id +WHERE mp.variant_key = 'unknown' + AND EXISTS (SELECT 1 FROM rate.master_products s WHERE s.item_id = mp.item_id + AND s.variant_key ~ '^[0-9]+(gb|tb)$' AND s.status <> 'archived'); + +-- V2 Every variant and every listing has an Item (M4 preconditions). expect 0 +SELECT (SELECT count(*) FROM rate.master_products WHERE item_id IS NULL) + + (SELECT count(*) FROM rate.products WHERE item_id IS NULL) AS without_item; + +-- V3 A listing's variant belongs to its Item. expect 0 +SELECT count(*) AS listing_item_mismatch +FROM rate.products p JOIN rate.master_products mp ON mp.id = p.cluster_id +WHERE mp.item_id IS DISTINCT FROM p.item_id; + +-- V4 Every Item with a live variant names a default variant of its own. expect 0 +SELECT count(*) AS default_variant_missing +FROM rate.items i +WHERE EXISTS (SELECT 1 FROM rate.master_products m WHERE m.item_id = i.id AND m.status <> 'archived') + AND NOT EXISTS (SELECT 1 FROM rate.master_products m + WHERE m.id = i.default_variant_id AND m.item_id = i.id); + +-- V5 Earbuds and other single-variant Items still have their variant (M3 never retires them). expect 0 +SELECT count(*) AS single_variant_retired +FROM rate.items i +LEFT JOIN rate.categories c ON c.id = i.category_id +WHERE (c.slug = 'earbuds' OR NOT EXISTS ( + SELECT 1 FROM rate.master_products s + WHERE s.item_id = i.id AND s.variant_key ~ '^[0-9]+(gb|tb)$')) + AND EXISTS (SELECT 1 FROM rate.m3_unknown_variant_backup b + WHERE b.item_id = i.id AND b.listing_id IS NULL); + +-- V6 (information) M3's work: variants retired and listings moved to "no variant". +SELECT (SELECT count(*) FROM rate.m3_unknown_variant_backup WHERE listing_id IS NULL) AS variants_retired, + (SELECT count(*) FROM rate.m3_unknown_variant_backup WHERE listing_id IS NOT NULL) AS listings_moved, + (SELECT count(*) FROM rate.products WHERE cluster_id IS NULL AND item_id IS NOT NULL) AS no_variant_now; + +-- V7 (information) Price alerts left on retired unknown variants (an alert needs a size). +SELECT mp.model_key, count(a.id) AS alerts +FROM rate.master_products mp +JOIN rate.price_alerts a ON a.master_product_id = mp.id +WHERE mp.variant_key = 'unknown' AND mp.status = 'archived' +GROUP BY 1 ORDER BY 2 DESC; diff --git a/Docs/Proposals/2026-10-29-dos-me-m3-m4.md b/Docs/Proposals/2026-10-29-dos-me-m3-m4.md new file mode 100644 index 000000000..d7efd6ca5 --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m3-m4.md @@ -0,0 +1,52 @@ +# Proposal: Item catalogue RPCs, and DOS-Me M3 + M4 (DOS/Rate#1160) + +**Status:** proposals for JOY, **not applied**. Zero DDL on prod from DOS/Rate; JOY applies them through DOS-Me (preview, beta, prod). Design: [`2026-10-27-item-variant-listing-model.md`](./2026-10-27-item-variant-listing-model.md), sections 4.2, 4.3, 5.1, 7, 7.2, 7.3. + +## Files + +| File | What | When | +| :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------- | +| [`2026-10-29-dos-me-item-rpcs.sql`](./2026-10-29-dos-me-item-rpcs.sql) (+ [`-rollback`](./2026-10-29-dos-me-item-rpcs-rollback.sql)) | `search_items_v1`, `get_category_brand_facets_v2`, `search_catalog_autocomplete_v2`; new functions only | any time after M1+M2 | +| [`2026-10-29-dos-me-m3-m4.sql`](./2026-10-29-dos-me-m3-m4.sql) (+ [`-rollback`](./2026-10-29-dos-me-m3-m4-rollback.sql), [`-verify`](./2026-10-29-dos-me-m3-m4-verify.sql)) | M3 (retire `unknown` on multi-size Items, with a backup table) and M4 (constraints, guard, default variants), two transactions | after the compatibility window | +| [`scripts/test-m3-m4-item-rpcs-proposal.mjs`](../../scripts/test-m3-m4-item-rpcs-proposal.mjs) | CI replay: M0, M1+M2, RPCs twice (called as `anon` and `rate_worker`), M3/M4 twice, every check, constraints and guard, both rollbacks twice, re-apply | every CI run | + +## Order of operations + +1. M0 ([`2026-10-29-dos-me-m0-seed-rekey.sql`](./2026-10-29-dos-me-m0-seed-rekey.sql)), then the seed change. +2. M1+M2 ([`2026-10-28-dos-me-m1-m2-items.sql`](./2026-10-28-dos-me-m1-m2-items.sql)). +3. **Item RPCs** (this proposal), whenever the web's Item catalogue (#1155, #1275) should stop grouping variants in the web. +4. The **compatibility window** (below). +5. **M3**, verify (V1, V5, V6, V7), then **M4**, verify (V2, V3, V4) and the M1+M2 verification file. + +## Compatibility window: conditions before M3/M4 + +- `RATE_ITEM_MODEL` is on for the importer and the web, and has run for at least one promote cycle. +- Rate Services ships Item-aware code (design task 13): its auditors treat `cluster_id IS NULL` as "no variant", its refreshes are keyed by listing and Item, and its payloads carry `item_id`. M3 creates such listings in bulk. +- The M1+M2 verification file returns 0 on every expect-0 check (the reconciliation after the last writer changed). +- No listing without an Item (`SELECT count(*) FROM rate.products WHERE item_id IS NULL` = 0): M4 aborts otherwise. +- The sitemap has been re-crawled with Item URLs (about 30 days, design section 7). +- Active price alerts on `unknown` variants of multi-size Items (V7 before M3) are decided: an alert needs a size, so each is moved to a sized variant or closed. M3 aborts while one is active. +- The importer and the Rate Services writers are **paused** for M3 and for each rollback: M3 retires and backs up in one transaction, and a listing written in between would not be in the backup. + +## What the files do + +**Item RPCs** (SECURITY INVOKER except autocomplete, which is DEFINER like v1 because anon has no USAGE on the pg_trgm schema; all read published Items and published variants only; EXECUTE for `anon`, `authenticated`, `service_role`, `rate_worker`): + +- `search_items_v1(...)`: one row per published Item with at least one published variant matching the filters. Same filter names as `search_master_products` where they apply; `p_storage_gb` means "has a variant of that size"; `p_has_5g` filters variants; `p_search` reads the Item and its published variants (name, key, chip); price filters and sorts use `matching_min_price` (cheapest priced matching variant). Returns the Item card fields and ratings (`user_rating_weighted`, `expert_rating`, `average_rating`, review counts), `matching_min_price` / `matching_max_price`, the representative variant (`representative_variant_id` and `_slug`: the cheapest priced matching variant, else the first matching one, so the card links to the size the filter asked for), `chips` (one JSON object per published variant: `variant_id`, `variant_key`, `slug`, `storage_gb`, `min_price`, `matching`), `variant_keys`, `matching_variant_keys`, `default_variant_id` and `total_count` for paging. Limit capped at 200. +- `get_category_brand_facets_v2(p_category_id)`: brands counted by published Item. +- `search_catalog_autocomplete_v2(p_query, p_limit)`: v1's result shape with `entity_kind = 'item'`. + +**M3:** aborts if an earbuds master still has a sized key (apply M0 first) or an active price alert sits on an `unknown` variant it would retire. Then, for every Item with a live sized variant (never earbuds) and a live `unknown` variant (a NULL status counts as live), calls `rate.retire_item_unknown_variant` (M2): the listings keep their Item with `cluster_id = NULL`, the variant is archived, stats refresh. It then backs up the variant and exactly the listings the RPC marked (`metadata.retired_unknown_variant`) into `rate.m3_unknown_variant_backup` (RLS on, service role only). Single-variant Items are untouched. + +**M4**, two transactions: (1) aborts unless every variant and listing has an Item; then `master_products.item_id NOT NULL`, `products_item_id_not_null` added as a CHECK `NOT VALID`, the `products_variant_item_guard` trigger (a listing's variant belongs to its Item), `items.default_variant_id` filled where unset (cheapest priced published variant, else the first published, else the first live one) and the `master_products_fill_default_variant` trigger, which names a default for an Item that has none when a live variant is written; (2) `VALIDATE CONSTRAINT` on its own, which holds only SHARE UPDATE EXCLUSIVE, so the listings table stays readable and writable during the scan. + +## Deviations from the design (decisions for JOY) + +- **M4 keeps the M1 sync triggers.** The design drops them in M4. The importer still relies on `master_products_sync_item` to create a new Item when it seeds a master (the matcher's `itemIdOfMaster` reads the Item the trigger made), and the listing trigger also carries the hand-set-variant guard and the item-follows-variant rule. Dropping them waits until the importer inserts Items itself (a later step); the M4 guard is the backstop meanwhile. +- **No master-side guard on `item_id` changes.** The design adds a companion trigger rejecting a variant's Item change while it has listings; M1's `master_products_move_listings` already moves those listings in the same statement, and V3 checks the result. +- **The M1+M2 retire RPC now treats a NULL status as live** (`IS DISTINCT FROM 'archived'`), the same as M3; the M1+M2 proposal file is updated in this PR (still not applied). + +## Rollback + +- RPCs: drop the three functions (turn the web's Item catalogue off first). +- M3/M4 (writers paused): drops the fill trigger, the guard, the CHECK and the NOT NULL, then restores the retired variants' status and their listings from the backup table (only listings still without a variant, on the same Item, still carrying the retire marker for that variant; any other listing changed after M3 and is left alone, with a NOTICE giving the count), refreshes stats, and drops the backup table. `default_variant_id` stays (nullable data M1+M2 already allowed). diff --git a/Docs/Proposals/2026-10-29-dos-me-m3-m4.sql b/Docs/Proposals/2026-10-29-dos-me-m3-m4.sql new file mode 100644 index 000000000..4c8a7b243 --- /dev/null +++ b/Docs/Proposals/2026-10-29-dos-me-m3-m4.sql @@ -0,0 +1,191 @@ +-- PROPOSAL, NOT APPLIED. DOS-Me migrations M3 (retire `unknown` on multi-size Items) and M4 +-- (constraints) for the Item model (DOS/Rate#1160; design +-- Docs/Proposals/2026-10-27-item-variant-listing-model.md sections 4.2, 4.3, 7, 7.2). +-- Applies only AFTER the compatibility window (see 2026-10-29-dos-me-m3-m4.md): M0 and M1+M2 +-- applied, every writer Item-aware (importer flag on, Rate Services task 13 handling NULL +-- cluster_id), and the M1+M2 reconciliation queries at zero. Run M3 with the importer and +-- Rate Services writers paused. JOY applies it through DOS-Me. +-- Rollback: 2026-10-29-dos-me-m3-m4-rollback.sql. Verification: +-- 2026-10-29-dos-me-m3-m4-verify.sql. +-- +-- Separate transactions (M3; M4 constraints; M4 validation) so each step can ship and be +-- verified on its own; every step aborts with a clear message when a precondition does not +-- hold, and every step is idempotent. + +-- M3 --------------------------------------------------------------------------------------- +BEGIN; + +-- What M3 changed, for the rollback (design 7.2): one row per retired variant (listing_id +-- NULL) and one per listing moved off it. +CREATE TABLE IF NOT EXISTS rate.m3_unknown_variant_backup ( + id bigserial PRIMARY KEY, + item_id uuid NOT NULL, + master_id uuid NOT NULL, + master_status text, + listing_id uuid, + listing_cluster_id uuid, + listing_variant_src text, + retired_at timestamptz NOT NULL DEFAULT now() +); +ALTER TABLE rate.m3_unknown_variant_backup ENABLE ROW LEVEL SECURITY; +REVOKE ALL ON rate.m3_unknown_variant_backup FROM PUBLIC, anon, authenticated; +GRANT ALL ON rate.m3_unknown_variant_backup TO service_role; + +-- Preconditions (design 7.3): no earbuds master with a sized key (seed drift, fixed by M0), +-- and no price alert on an unknown variant M3 would archive (an alert needs a size; JOY +-- decides per alert before M3: move it to a sized variant or close it). The web keeps +-- running, so price_alerts is locked against writes (reads continue) until M3 commits: no +-- alert can appear between this check and the retire. +LOCK TABLE rate.price_alerts IN SHARE MODE; +DO $$ +DECLARE + v_sized_earbuds text; + v_alerted text; +BEGIN + SELECT string_agg(mp.model_key, ', ') INTO v_sized_earbuds + FROM rate.master_products mp JOIN rate.categories c ON c.id = mp.category_id + WHERE c.slug = 'earbuds' AND split_part(mp.model_key, ':', 3) <> 'unknown'; + IF v_sized_earbuds IS NOT NULL THEN + RAISE EXCEPTION 'M3: earbuds masters with a sized key (apply M0 first): %', v_sized_earbuds; + END IF; + SELECT string_agg(DISTINCT u.model_key, ', ') INTO v_alerted + FROM rate.master_products u + JOIN rate.price_alerts a ON a.master_product_id = u.id AND a.is_active + WHERE u.variant_key = 'unknown' AND u.status IS DISTINCT FROM 'archived' + AND EXISTS (SELECT 1 FROM rate.master_products s + WHERE s.item_id = u.item_id AND s.status IS DISTINCT FROM 'archived' + AND s.variant_key ~ '^[0-9]+(gb|tb)$'); + IF v_alerted IS NOT NULL THEN + RAISE EXCEPTION 'M3: price alerts on unknown variants to retire, decide them first: %', v_alerted; + END IF; +END $$; + +-- Every multi-size Item (a live sized variant, never earbuds) still holding a live unknown +-- variant: retire it through the M2 RPC (same lock order and markers as the importer's +-- calls), then back up exactly what the RPC moved, read from its markers (they include the +-- listings its second sweep caught). +DO $$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT i.id AS item_id, u.id AS master_id, u.status + FROM rate.items i + JOIN rate.master_products u + ON u.item_id = i.id AND u.variant_key = 'unknown' AND u.status IS DISTINCT FROM 'archived' + LEFT JOIN rate.categories c ON c.id = i.category_id + WHERE COALESCE(c.slug, '') <> 'earbuds' + AND EXISTS (SELECT 1 FROM rate.master_products s + WHERE s.item_id = i.id AND s.status IS DISTINCT FROM 'archived' + AND s.variant_key ~ '^[0-9]+(gb|tb)$') + ORDER BY i.id + LOOP + PERFORM rate.retire_item_unknown_variant(r.item_id); + IF (SELECT status FROM rate.master_products WHERE id = r.master_id) = 'archived' THEN + INSERT INTO rate.m3_unknown_variant_backup (item_id, master_id, master_status) + VALUES (r.item_id, r.master_id, r.status); + INSERT INTO rate.m3_unknown_variant_backup + (item_id, master_id, master_status, listing_id, listing_cluster_id, listing_variant_src) + SELECT r.item_id, r.master_id, r.status, p.id, r.master_id, + p.metadata ->> 'variant_source_before_retire' + FROM rate.products p + WHERE p.metadata ->> 'retired_unknown_variant' = r.master_id::text; + END IF; + END LOOP; +END $$; + +COMMIT; + +-- M4 (constraints) --------------------------------------------------------------------------- +BEGIN; + +-- Preconditions: every variant and every listing has an Item (a listing that never matched +-- must be matched or archived first, design 7.1). +DO $$ +DECLARE + v_masters bigint; + v_listings bigint; +BEGIN + SELECT count(*) INTO v_masters FROM rate.master_products WHERE item_id IS NULL; + SELECT count(*) INTO v_listings FROM rate.products WHERE item_id IS NULL; + IF v_masters > 0 OR v_listings > 0 THEN + RAISE EXCEPTION 'M4: % master(s) and % listing(s) without item_id; reconcile first', v_masters, v_listings; + END IF; +END $$; + +-- master_products is small (one row per variant): the NOT NULL scan is short. +ALTER TABLE rate.master_products ALTER COLUMN item_id SET NOT NULL; +-- The listings CHECK is added NOT VALID here (no scan) and validated in its own transaction +-- below, which takes only SHARE UPDATE EXCLUSIVE, so reads and writes keep running. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'rate.products'::regclass AND conname = 'products_item_id_not_null') THEN + ALTER TABLE rate.products ADD CONSTRAINT products_item_id_not_null CHECK (item_id IS NOT NULL) NOT VALID; + END IF; +END $$; + +-- A listing's variant must belong to the listing's Item (design 4.3). The M1 sync trigger +-- (trg_products_sync_item, which runs first by name) already derives item_id from the +-- variant; this guard is the backstop for any write that gets around it. +CREATE OR REPLACE FUNCTION rate.products_variant_item_guard() +RETURNS trigger +LANGUAGE plpgsql +SET search_path = '' +AS $$ +BEGIN + IF NEW.cluster_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM rate.master_products mp + WHERE mp.id = NEW.cluster_id AND mp.item_id = NEW.item_id) THEN + RAISE EXCEPTION 'listing % variant % is not a variant of item %', NEW.id, NEW.cluster_id, NEW.item_id + USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END $$; +REVOKE EXECUTE ON FUNCTION rate.products_variant_item_guard() FROM PUBLIC, anon, authenticated; +DROP TRIGGER IF EXISTS trg_products_variant_item_guard ON rate.products; +CREATE TRIGGER trg_products_variant_item_guard + BEFORE INSERT OR UPDATE OF cluster_id, item_id ON rate.products + FOR EACH ROW EXECUTE FUNCTION rate.products_variant_item_guard(); + +-- Every Item names its default variant: the cheapest priced published one, else the first +-- published one, else the first live one (by sort_order, storage, key). Only where unset. +UPDATE rate.items i +SET default_variant_id = d.id +FROM ( + SELECT DISTINCT ON (m.item_id) m.item_id, m.id + FROM rate.master_products m + WHERE m.status IS DISTINCT FROM 'archived' + ORDER BY m.item_id, + (m.status = 'published') DESC NULLS LAST, + (m.min_price > 0) DESC NULLS LAST, + m.min_price ASC NULLS LAST, + m.sort_order NULLS LAST, m.storage_gb NULLS LAST, m.variant_key, m.id +) d +WHERE d.item_id = i.id AND i.default_variant_id IS NULL; + +-- ...and an Item created later (or whose default was cleared) gets its first live variant +-- as default when that variant is written (lock order variant, then Item). +CREATE OR REPLACE FUNCTION rate.items_fill_default_variant() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + IF NEW.status IS DISTINCT FROM 'archived' THEN + UPDATE rate.items SET default_variant_id = NEW.id + WHERE id = NEW.item_id AND default_variant_id IS NULL; + END IF; + RETURN NULL; +END $$; +REVOKE EXECUTE ON FUNCTION rate.items_fill_default_variant() FROM PUBLIC, anon, authenticated; +DROP TRIGGER IF EXISTS trg_master_products_fill_default_variant ON rate.master_products; +CREATE TRIGGER trg_master_products_fill_default_variant + AFTER INSERT OR UPDATE OF status, item_id, model_key ON rate.master_products + FOR EACH ROW EXECUTE FUNCTION rate.items_fill_default_variant(); + +COMMIT; + +-- M4 (validation, own transaction: SHARE UPDATE EXCLUSIVE only) ------------------------------- +ALTER TABLE rate.products VALIDATE CONSTRAINT products_item_id_not_null; diff --git a/Docs/Proposals/2026-10-29-item-model-go-live.html b/Docs/Proposals/2026-10-29-item-model-go-live.html new file mode 100644 index 000000000..712d2d0fe --- /dev/null +++ b/Docs/Proposals/2026-10-29-item-model-go-live.html @@ -0,0 +1,199 @@ + + + + + + + Kế hoạch bật mô hình Item + + + +
+ Chưa áp dụng gì + Chờ JOY duyệt 9 quyết định +

Kế hoạch bật mô hình Item (sản phẩm, phiên bản, listing)

+

Ngày 29/10/2026 - Issue DOS/Rate#1150 - Bản tiếng Anh: 2026-10-29-item-model-go-live.md - Người soạn: session Core của Rate Core PO

+ +
+

Tóm tắt

+

Trang này gom toàn bộ các bước đưa mô hình Item lên thật vào một chỗ, để JOY duyệt một lần. Item là một sản phẩm (ví dụ iPhone 17 Pro), phiên bản (variant) là một dung lượng của nó (256GB), listing là một tin bán trên sàn. Có 9 bước theo thứ tự, mỗi bước ghi: thay đổi gì, kiểm tra thế nào (chỉ đọc), hoàn tác thế nào, ai chạy. Cuối trang là 9 quyết định JOY cần chốt, mỗi câu có khuyến nghị sẵn.

+

Mọi file SQL đã được chạy thử trong CI (kiểm thử tự động trên GitHub) trên một database tạm: chạy hai lần, kiểm tra, hoàn tác hai lần, chạy lại. Chưa bước nào đụng tới Supabase prod dùng chung.

+
+ +

1. Luật chung cho mọi bước database

+ + +

2. Thứ tự tổng quát

+
+ + + + + + + + + + + +
#BướcAi chạyPhải chờ
1#1123 / M0: đổi khóa (key) seed lệch chuẩn, chỉ sửa dữ liệuJOY qua DOS-Mequyết định 8
2PR sửa seed (flagship-seed-data.ts), ngay sau M0Core mở PR, PO mergebước 1 đã lên prod
3M1+M2: bảng rate.items, cột mới, trigger, RPC, điền dữ liệuJOY qua DOS-Mebước 2; quyết định 1 đến 5
4RPC danh mục Item (tìm kiếm, đếm theo hãng, gợi ý khi gõ)JOY qua DOS-Mebước 3
5Rate Services nâng phiên bản vendor/rateRate Services (RS PO mở PR), JOY mergebước 3
6Bật cờ RATE_ITEM_MODEL=item, rồi nullable_variantBiến môi trường importer (Rate Services), biến môi trường Vercel (web)bước 4 và 5
7Giai đoạn tương thíchmọi bên; PO theo dõibước 6, ít nhất một vòng promote và khoảng 30 ngày
8M3: gỡ phiên bản "unknown" ở Item nhiều dung lượngJOY qua DOS-Me, dừng tiến trình ghibước 7; quyết định 4
9M4: ràng buộc và phiên bản mặc địnhJOY qua DOS-Mebước 8; quyết định 6, 7
+
+
+

Lưu ý về thứ tự. #1123 và M0 là một bước: M0 là file SQL đã review cho #1123, cộng thêm ba khóa tai nghe lệch chuẩn khác. PR sửa seed phải merge ngay sau M0, trước M1+M2. Giữa M0 và PR seed tuyệt đối không chạy jobs/backfill-specs.ts --apply: job này tìm sản phẩm theo khóa seed cũ, không thấy sẽ tạo mới, sinh ra bản trùng.

+
+ +

3. Chi tiết từng bước

+ +

Bước 1. #1123 / M0: đổi khóa seed

+
+
Thay đổi
Sáu sản phẩm seed bị lệch được đổi sang khóa chuẩn của bộ chuẩn hóa (normalizer), sửa tại chỗ: id, slug, listing, cảnh báo giá, đánh giá giữ nguyên. Tai nghe đặt storage_gb = NULL. Ghi dấu vào metadata.m0_rekeyed_from và m0_storage_gb_before để hoàn tác. Nếu khóa đích đã có người dùng thì bỏ qua và báo NOTICE, không bao giờ tự gộp.
+
File
2026-10-29-dos-me-m0-seed-rekey.sql (chỉ dữ liệu, một transaction, chạy lại an toàn).
+
Kiểm tra
Trước khi chạy: V5 trong 2026-10-29-dos-me-m0-seed-rekey-verify.sql, so với bảng sáu dòng trong đề xuất M0. Sau khi chạy: V1 đến V4 trả về 0.
+
Hoàn tác
2026-10-29-dos-me-m0-seed-rekey-rollback.sql, đồng thời revert PR seed. Chỉ làm được khi M1+M2 chưa áp dụng (nếu đã áp dụng thì hoàn tác M1+M2 trước).
+
Ai chạy
JOY qua DOS-Me.
+
+ +

Bước 2. PR sửa seed

+
+
Thay đổi
apps/importer/src/shopee/flagship-seed-data.ts dùng sáu khóa mới; tai nghe storage_gb: null. Core mở PR khi JOY lên lịch M0; không được merge trước khi M0 lên prod.
+
Kiểm tra
CI xanh; truy vấn chỉ đọc xác nhận không có model_key nào xuất hiện hai lần trong rate.master_products.
+
Hoàn tác
Revert PR (cùng lúc với hoàn tác M0).
+
Ai chạy
Core mở, PO merge.
+
+ +

Bước 3. M1+M2: bảng Item

+
+
Thay đổi
Bảng rate.items (RLS, tức phân quyền theo dòng: ai cũng đọc được Item đã xuất bản; rate_worker đọc tất cả, chỉ được sửa cột thông số). Cột mới master_products.item_id / variant_key / sort_order và products.item_id / variant_source. Trigger đồng bộ: sản phẩm mới tự có Item theo khóa, listing đi theo. RPC match_item, refresh_item_stats, retire_item_unknown_variant; hàm điểm đánh giá và lượt vote hiểu Item nhưng giữ nguyên chữ ký. Điền dữ liệu cho bảng mới. Một transaction, chạy lại an toàn.
+
File
2026-10-28-dos-me-m1-m2-items.sql.
+
Kiểm tra
2026-10-28-dos-me-m1-m2-items-verify.sql, V1 đến V15, chạy trước và sau. Về sau file này cũng dùng làm bước đối soát.
+
Hoàn tác
2026-10-28-dos-me-m1-m2-items-rollback.sql, dừng tiến trình ghi và tắt cờ về off trước. Khôi phục thân cũ của hai hàm điểm đánh giá. Không bao giờ TRUNCATE rate.items.
+
Ai chạy
JOY qua DOS-Me.
+
+ +

Bước 4. RPC danh mục Item

+
+
Thay đổi
Chỉ thêm hàm mới: search_items_v1, get_category_brand_facets_v2, search_catalog_autocomplete_v2; quyền gọi cho anon, authenticated, service_role, rate_worker. Các RPC cũ mà web dùng khi tắt cờ không bị đụng tới.
+
File
2026-10-29-dos-me-item-rpcs.sql.
+
Kiểm tra (chỉ đọc)
Với vai trò anon: SELECT count(*) FROM rate.search_items_v1(p_limit => 200) lớn hơn 0, và total_count bằng số Item đã xuất bản có ít nhất một phiên bản đã xuất bản. SELECT * FROM rate.search_catalog_autocomplete_v2('iphone', 5) trả các dòng entity_kind = 'item'.
+
Hoàn tác
Tắt cờ web, rồi chạy 2026-10-29-dos-me-item-rpcs-rollback.sql.
+
Ai chạy
JOY qua DOS-Me.
+
+ +

Bước 5. Rate Services nâng vendor/rate

+
+
Thay đổi
Rate Services chuyển submodule vendor/rate sang một commit dev của DOS/Rate có importer hiểu Item (#1179, #1190, #1208, #1213 và các PR sau). Phần việc Item của chính Rate Services đã merge: RS#685 (cluster_id NULL nghĩa là chưa rõ phiên bản), RS#687 (xác minh theo Item, job mang id Item, làm mới theo Item), RS#689.
+
Kiểm tra
CI của Rate Services xanh; khi cờ vẫn off, một vòng promote bình thường không phát sinh lỗi mới.
+
Hoàn tác
Trả submodule về commit cũ.
+
Ai chạy
Rate Services (RS PO mở PR), JOY merge.
+
+ +

Bước 6. Bật cờ

+

RATE_ITEM_MODEL có ba giá trị: off (mặc định; giá trị lạ cũng hiểu là off), item (ghép theo Item trước, ghi item_id; listing nào cũng vẫn có phiên bản), nullable_variant (như item, thêm: listing không ghi dung lượng thuộc Item nhiều dung lượng được ghi cluster_id = NULL).

+
+
Thứ tự
Importer item, rồi web item (Vercel Preview trước, kiểm tra, rồi Production), rồi importer nullable_variant.
+
Kiểm tra
Sau mỗi lần đổi: file kiểm tra M1+M2 trả 0 ở mọi dòng expect 0. Trên web, URL phiên bản cũ chuyển hướng 308 sang trang Item (job E2E Item trong CI đã kiểm tra việc này).
+
Hoàn tác
Đặt lại biến môi trường về off và deploy lại. Dữ liệu ghi trong lúc bật vẫn hợp lệ khi tắt (trigger đồng bộ giữ các cột cũ khớp nhau).
+
Ai chạy
Biến môi trường importer ở Rate Services; biến môi trường web ở Vercel (JOY).
+
+ +

Bước 7. Giai đoạn tương thích (điều kiện trước M3/M4)

+ + +

Bước 8. M3: gỡ phiên bản "unknown" ở Item nhiều dung lượng

+
+
Thay đổi
Tự dừng nếu còn tai nghe mang khóa có dung lượng, hoặc còn cảnh báo giá đang bật trên phiên bản "unknown" sắp gỡ. Bảng price_alerts bị khóa ghi trong lúc chạy. Sau đó lưu trữ (archive) phiên bản "unknown" còn sống của mọi Item nhiều dung lượng qua retire_item_unknown_variant; listing của nó giữ Item nhưng không có phiên bản; số liệu được tính lại. Bảng sao lưu rate.m3_unknown_variant_backup (bật RLS, chỉ service role đọc) ghi đúng những gì đã đổi. Item một dung lượng (tai nghe, dòng không có dung lượng) không bị đụng.
+
File
2026-10-29-dos-me-m3-m4.sql, transaction đầu. Dừng tiến trình ghi.
+
Kiểm tra
2026-10-29-dos-me-m3-m4-verify.sql: V1 và V5 trả 0; V6 (việc đã làm) và V7 (cảnh báo giá) để tham khảo.
+
Hoàn tác
2026-10-29-dos-me-m3-m4-rollback.sql, dừng tiến trình ghi: khôi phục phiên bản và chỉ những listing còn mang dấu của M3 (NOTICE báo số listing đã đổi sau đó), tính lại số liệu, xóa bảng sao lưu.
+
Ai chạy
JOY qua DOS-Me.
+
+ +

Bước 9. M4: ràng buộc

+
+
Thay đổi
Tự dừng nếu còn phiên bản hoặc listing thiếu Item. Sau đó: master_products.item_id NOT NULL; ràng buộc CHECK products_item_id_not_null (thêm ở dạng NOT VALID, kiểm tra trong transaction riêng nên bảng listing vẫn đọc ghi được); trigger chặn listing gắn phiên bản của Item khác; điền items.default_variant_id, kèm trigger tự điền cho Item mới.
+
File
2026-10-29-dos-me-m3-m4.sql, transaction thứ hai và thứ ba.
+
Kiểm tra
V2, V3, V4 trong file kiểm tra M3/M4 trả 0, và chạy lại file kiểm tra M1+M2.
+
Hoàn tác
Transaction đầu của file hoàn tác M3/M4 chỉ gỡ M4; transaction thứ hai mới gỡ M3.
+
Ai chạy
JOY qua DOS-Me.
+
+ +

4. Các quyết định JOY cần chốt

+

Mỗi câu có khuyến nghị; nếu JOY đồng ý thì khuyến nghị là mặc định.

+ +
1. (M1+M2) match_item lọc theo hãng, không theo model. Thiết kế có tham số model, nhưng lọc theo model thì chỉ tìm được đúng model đó, việc mà bước ghép chính xác (Tier 1) đã làm rồi. Khuyến nghị: lọc theo hãng.
+
2. (M1+M2) Item chưa có đánh giá lưu điểm 0, giống các cột của phiên bản hiện nay; web hiện "chưa có đánh giá" khi số đánh giá bằng 0. Khuyến nghị: giữ 0.
+
3. (M1+M2) Không sửa view rate.v_products_with_merchant_trust. Thêm item_id vào view làm hỏng việc chạy lại migration 20261018153000; ai cần item_id thì đọc từ rate.products. Khuyến nghị: giữ nguyên view.
+
4. (M1+M2, M3 cập nhật) Cảnh báo giá trên phiên bản "unknown". Cảnh báo giá cần một dung lượng cụ thể. M1+M2 để nguyên cảnh báo trên phiên bản đó; M3 giờ từ chối chạy khi còn cảnh báo đang bật, nên trước M3 JOY xử lý từng cái: chuyển sang phiên bản có dung lượng hoặc đóng. Khuyến nghị: xử lý từng cảnh báo theo V7 trước M3.
+
5. (M1+M2) Tên công tắc cho admin: rate.variant_override. Công cụ admin nào gán lại phiên bản đã chọn tay thì bật công tắc này trong transaction của nó. Khuyến nghị: duyệt tên này.
+
6. (M3/M4) M4 giữ lại các trigger đồng bộ của M1. Thiết kế định bỏ chúng ở M4, nhưng importer vẫn dựa vào chúng để tạo Item mới khi seed sản phẩm. Trigger chặn của M4 làm lớp bảo vệ cuối; bỏ trigger đồng bộ để sau, khi importer tự tạo Item. Khuyến nghị: giữ lại.
+
7. (M3/M4) Không thêm trigger chặn phía phiên bản khi đổi Item của nó. Trigger dời listing của M1 đã dời listing ngay trong cùng câu lệnh, và V3 kiểm tra kết quả. Khuyến nghị: không thêm.
+
8. (M0) Duyệt việc ghi dữ liệu M0, kể cả gộp apple:airpods-pro-2-usbc thành Item apple:airpods-pro-2. Bộ chuẩn hóa không tách bản USB-C với Lightning; hàng mới hiện chỉ còn bản USB-C. Khuyến nghị: duyệt.
+
9. (CI) Đặt job E2E (Playwright, Item model) thành check bắt buộc trên dev. Job chạy xanh, không cần chạy lại, ở mọi lần CI của PR #1273 (khoảng 4 phút). Đây là cài đặt bảo vệ nhánh (branch protection) của repo. Khuyến nghị: bắt buộc.
+
+ + diff --git a/Docs/Proposals/2026-10-29-item-model-go-live.md b/Docs/Proposals/2026-10-29-item-model-go-live.md new file mode 100644 index 000000000..8b21df290 --- /dev/null +++ b/Docs/Proposals/2026-10-29-item-model-go-live.md @@ -0,0 +1,124 @@ +# Runbook: Item model go-live (DOS/Rate#1150) + +**Status:** runbook for JOY, **nothing applied**. One page to approve the whole Item / variant / listing rollout in one pass. Vietnamese version for JOY: [`2026-10-29-item-model-go-live.html`](./2026-10-29-item-model-go-live.html). + +Design (approved 2026-10-27, #1129): [`2026-10-27-item-variant-listing-model.md`](./2026-10-27-item-variant-listing-model.md). Per-step proposals: + +- M0: [`2026-10-29-dos-me-m0-seed-rekey.md`](./2026-10-29-dos-me-m0-seed-rekey.md) (#1212, #1123) +- M1+M2: [`2026-10-28-dos-me-m1-m2-items.md`](./2026-10-28-dos-me-m1-m2-items.md) (#1150) +- Item RPCs, M3, M4: [`2026-10-29-dos-me-m3-m4.md`](./2026-10-29-dos-me-m3-m4.md) (#1160, PR #1279) + +Every SQL file is replayed in CI on a throwaway PostgreSQL 17 stack (apply twice, checks, rollback twice, re-apply). No step has touched the shared prod Supabase. + +## Rules for every database step + +- **JOY applies it through DOS-Me**: preview, then beta, then prod, verifying after each. DOS/Rate never runs DDL or data writes on prod. The DOS-Me beta flow is still locked (ROADMAP 6.4); until it reopens, prod waits. +- **Verification files are read-only** (`SELECT` only). Every check marked "expect 0" must return 0. +- After each prod apply, mirror the applied file into `supabase/migrations/` and regenerate `database.types.ts` (a Rate PR, as #1147 did). +- **Writers paused** means: the importer jobs and the Rate Services workers stopped for the duration (the web keeps running). +- Rollbacks run in reverse order of the steps applied. + +## Order at a glance + +| # | Step | Who runs it | Blocks on | +| :-- | :--------------------------------------------------- | :--------------------------------------------- | :---------------------------------------------- | +| 1 | #1123 / M0: seed key re-key (data) | JOY via DOS-Me | decision 8 | +| 2 | Seed PR (`flagship-seed-data.ts`), right after M0 | Core opens, PO merges | step 1 on prod | +| 3 | M1+M2: `rate.items`, columns, triggers, RPCs, fill | JOY via DOS-Me | step 2; decisions 1 to 5 | +| 4 | Item catalogue RPCs | JOY via DOS-Me | step 3 | +| 5 | Rate Services bump of `vendor/rate` | Rate Services (RS PO), JOY merges | step 3 | +| 6 | Flag `RATE_ITEM_MODEL=item`, then `nullable_variant` | Importer env (Rate Services), Vercel env (web) | steps 4 and 5 | +| 7 | Compatibility window | everyone; PO tracks | step 6, at least one promote cycle and ~30 days | +| 8 | M3: retire `unknown` on multi-size Items | JOY via DOS-Me, writers paused | step 7; decision 4 | +| 9 | M4: constraints and default variants | JOY via DOS-Me | step 8; decisions 6, 7 | + +Note on the order: #1123 and M0 are one step (M0 is the reviewed SQL for #1123, plus three more drifted earbuds keys). The seed PR must merge **right after M0, before M1+M2**, and `jobs/backfill-specs.ts --apply` must not run between M0 and the seed PR: it looks masters up by seed key and would re-create the old keys as duplicates. + +## Steps + +### 1. #1123 / M0: seed key re-key + +- **Changes:** six drifted seed masters get the normalizer's keys in place (ids, slugs, listings, alerts, reviews unchanged); earbuds masters get `storage_gb = NULL`; markers `metadata.m0_rekeyed_from`, `m0_storage_gb_before`. A target key already taken is skipped with a NOTICE (never an automatic merge). +- **File:** `2026-10-29-dos-me-m0-seed-rekey.sql` (data only, one transaction, idempotent). +- **Verify:** before, run V5 of `2026-10-29-dos-me-m0-seed-rekey-verify.sql` and compare with the six rows in the M0 proposal; after, V1 to V4 return 0. +- **Roll back:** `2026-10-29-dos-me-m0-seed-rekey-rollback.sql` and revert the seed PR in the same step. Only while M1+M2 is not applied (roll M1+M2 back first otherwise). +- **Who:** JOY through DOS-Me. + +### 2. Seed PR + +- **Changes:** `apps/importer/src/shopee/flagship-seed-data.ts` uses the six new keys; earbuds `storage_gb: null`. The Core session opens it once JOY schedules M0; it must not merge before M0 is on prod. +- **Verify:** CI green; read-only check that no `model_key` appears twice in `rate.master_products`. +- **Roll back:** revert the PR (together with the M0 rollback). +- **Who:** Core opens, PO merges. + +### 3. M1+M2: Items + +- **Changes:** `rate.items` (RLS: public reads published; `rate_worker` reads all, updates spec columns only), `master_products.item_id / variant_key / sort_order`, `products.item_id / variant_source`, sync triggers (a master gets its Item from its key; listings follow), RPCs `match_item`, `refresh_item_stats`, `retire_item_unknown_variant`, Item-aware rating and vote functions with the same signatures, and the backfill. One transaction, idempotent. +- **File:** `2026-10-28-dos-me-m1-m2-items.sql`. +- **Verify:** `2026-10-28-dos-me-m1-m2-items-verify.sql`, V1 to V15, before and after (the same file is the reconciliation later). +- **Roll back:** `2026-10-28-dos-me-m1-m2-items-rollback.sql`, writers paused and the flags `off` first. Restores the two rating functions' previous bodies. Never `TRUNCATE rate.items`. +- **Who:** JOY through DOS-Me. + +### 4. Item catalogue RPCs + +- **Changes:** new functions only: `search_items_v1`, `get_category_brand_facets_v2`, `search_catalog_autocomplete_v2`; EXECUTE for `anon`, `authenticated`, `service_role`, `rate_worker`. The variant RPCs the flag-off web uses are untouched. +- **File:** `2026-10-29-dos-me-item-rpcs.sql`. +- **Verify (read-only):** as `anon`, `SELECT count(*) FROM rate.search_items_v1(p_limit => 200)` is non-zero and its `total_count` equals `SELECT count(*) FROM rate.items i WHERE i.status = 'published' AND EXISTS (SELECT 1 FROM rate.master_products m WHERE m.item_id = i.id AND m.status = 'published')`; `SELECT * FROM rate.search_catalog_autocomplete_v2('iphone', 5)` returns `entity_kind = 'item'` rows. +- **Roll back:** turn the web flag off, then `2026-10-29-dos-me-item-rpcs-rollback.sql`. +- **Who:** JOY through DOS-Me. + +### 5. Rate Services bump + +- **Changes:** Rate Services moves its `vendor/rate` pin to a DOS/Rate `dev` commit with the Item-aware importer (#1179, #1190, #1208, #1213 and later). Its own Item work already merged (RS#685 NULL `cluster_id` is "no variant", RS#687 Item verification, job Item ids, Item refresh, RS#689). +- **Verify:** Rate Services CI green on the bump; with the flag still `off`, one normal promote cycle shows no new errors. +- **Roll back:** revert the pin. +- **Who:** Rate Services (RS PO opens the PR), JOY merges. + +### 6. Flags + +`RATE_ITEM_MODEL` has three values: `off` (default; any unknown value reads as `off`), `item` (Item-first matching, writes `item_id`; every listing still has a variant), `nullable_variant` (as `item`, and a listing with no stated size on a multi-size Item gets `cluster_id = NULL`). + +- **Order:** importer `item` -> web `item` (Vercel Preview, check, then Production) -> importer `nullable_variant`. +- **Verify:** after each switch, the M1+M2 verification file returns 0 on every expect-0 check; on the web, an old variant URL 308-redirects to its Item page (the Item E2E job covers this in CI). +- **Roll back:** set the env back to `off` and redeploy; data written meanwhile stays valid under `off` (the sync triggers keep the old columns consistent). +- **Who:** importer env in Rate Services; web env in Vercel (JOY). + +### 7. Compatibility window (conditions before M3/M4) + +- `RATE_ITEM_MODEL` on for the importer and the web for at least one promote cycle. +- Rate Services Item-aware code shipped (step 5). +- M1+M2 verification at 0 after the last writer changed (the reconciliation). +- No listing without an Item: `SELECT count(*) FROM rate.products WHERE item_id IS NULL` = 0 (M4 aborts otherwise). +- Sitemap re-crawled with Item URLs (about 30 days). +- Active price alerts on `unknown` variants of multi-size Items decided (V7 of the M3/M4 verify file): each moved to a sized variant or closed. M3 aborts while one is active. +- **Roll back the window:** flags `off`. + +### 8. M3: retire `unknown` on multi-size Items + +- **Changes:** aborts on a sized earbuds key or an active alert on an `unknown` to retire (`price_alerts` locked against writes while it runs). Then archives the live `unknown` variant of every multi-size Item through `retire_item_unknown_variant`; its listings keep their Item with no variant; stats refresh. Backup table `rate.m3_unknown_variant_backup` (RLS on, service role only) records exactly what changed. Single-size Items (earbuds, sizeless lines) untouched. +- **File:** `2026-10-29-dos-me-m3-m4.sql`, first transaction. Writers paused. +- **Verify:** `2026-10-29-dos-me-m3-m4-verify.sql` V1 and V5 return 0; V6 (work done) and V7 (alerts) for information. +- **Roll back:** `2026-10-29-dos-me-m3-m4-rollback.sql`, writers paused: restores the variants and only the listings still carrying the retire marker (a NOTICE counts any changed since), refreshes stats, drops the backup table. +- **Who:** JOY through DOS-Me. + +### 9. M4: constraints + +- **Changes:** aborts unless every variant and listing has an Item; `master_products.item_id NOT NULL`; `products_item_id_not_null` CHECK (added NOT VALID, validated in its own transaction so the listings table stays readable and writable); guard trigger (a listing's variant belongs to its Item); `items.default_variant_id` filled, plus a trigger filling it for new Items. +- **File:** `2026-10-29-dos-me-m3-m4.sql`, second and third transactions. +- **Verify:** M3/M4 verify V2, V3, V4 return 0, and the M1+M2 verification file again. +- **Roll back:** the first transaction of the same rollback file undoes M4 alone; its second transaction then undoes M3. +- **Who:** JOY through DOS-Me. + +## Open decisions for JOY + +Each has a recommendation, which is the default if JOY agrees. + +1. **(M1+M2) `match_item` filters by brand, not by model.** The design names a model argument; filtering by it only finds the exact model, which Tier 1 already does. **Recommend: brand.** +2. **(M1+M2) An Item with no reviews stores rating 0**, like the variant columns today; the web shows "no rating" when the review count is 0. **Recommend: keep 0.** +3. **(M1+M2) `rate.v_products_with_merchant_trust` is not changed.** Adding `item_id` breaks a re-run of `20261018153000`; readers take `item_id` from `rate.products`. **Recommend: leave the view.** +4. **(M1+M2, updated by M3) Price alerts on an `unknown` variant.** An alert needs a size. M1+M2 leaves them on the variant; M3 now refuses to run while one is active, so JOY decides each one before M3: move it to a sized variant or close it. **Recommend: decide per alert from V7 before M3.** +5. **(M1+M2) Admin override name `rate.variant_override`.** Any admin tool that re-assigns a hand-set variant sets it locally. **Recommend: approve the name.** +6. **(M3/M4) M4 keeps the M1 sync triggers.** The design drops them; the importer still relies on them to create a new Item when it seeds a master. The M4 guard is the backstop; dropping them waits until the importer inserts Items itself. **Recommend: keep them.** +7. **(M3/M4) No master-side guard on a variant's Item change.** M1's move-listings trigger moves the listings in the same statement, and V3 checks the result. **Recommend: no extra guard.** +8. **(M0) Approve the M0 data write, including `apple:airpods-pro-2-usbc` becoming Item `apple:airpods-pro-2`** (the normalizer does not split USB-C from Lightning; only USB-C is sold new). **Recommend: approve.** +9. **(CI) Make `E2E (Playwright, Item model)` a required check on `dev`.** It ran green with no retry on every run of PR #1273 (about 4 minutes). Branch protection is a repository setting. **Recommend: required.** diff --git a/apps/extension/CHROMEWEBSTORE.md b/apps/extension/CHROMEWEBSTORE.md index 93eb22a3e..9b6df0a42 100644 --- a/apps/extension/CHROMEWEBSTORE.md +++ b/apps/extension/CHROMEWEBSTORE.md @@ -14,7 +14,7 @@ Rate.vn - So Sánh Giá, Lịch Sử Giá & Lọc Đánh Giá Ảo AI ### Short Summary (<= 132 characters) -Kiểm tra lịch sử giá 90 ngày, phát hiện giảm giá ảo, tìm nơi bán rẻ hơn và lọc đánh giá seeding bằng AI trên Shopee, Lazada, Tiki. +Xem lịch sử giá 90 ngày, phát hiện giảm giá ảo, tìm nơi bán rẻ hơn, lọc đánh giá ảo bằng AI trên 10 sàn và chuỗi bán lẻ Việt Nam. ### Detailed Description @@ -30,21 +30,25 @@ Rate.vn giúp người tiêu dùng mua sắm đồ công nghệ minh bạch và ## 2. Permissions Justification (For Chrome Web Store Review Team) -| Permission / Host Permission | Plain-English Justification | -| :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `storage` | Saves the user's preferred Rate.vn server endpoint, temporary product lookup cache for the active session, and the user's explicit opt-in preference for contributing public product price signals. | -| `sidePanel` | Displays the detailed price history chart, cross-merchant price comparison table, and AI review authenticity breakdown alongside the shopping page when the user clicks the extension badge or floating pill. | -| `tabs` | Reads the active shopping tab's product URL and title so the Side Panel can display the matching price history and cross-store comparison for the product currently being viewed. | -| `https://shopee.vn/*`, `https://*.lazada.vn/*`, `https://tiki.vn/*`, `https://cellphones.com.vn/*`, `https://*.thegioididong.com/*`, `https://fptshop.com.vn/*` | Reads the public product title, selected storage variant, and listed price on supported Vietnamese e-commerce product pages to display the floating price-check badge. | -| `https://rate.vn/*`, `http://localhost:3000/*` | Fetches verified price history, cross-merchant offers, and review authenticity scores from the Rate.vn API. | +This table lists exactly what `manifest.json` requests; `test-extension-pipeline.ts` fails when the two disagree (DOS/Rate#1231). + +| Permission / Host Permission | Plain-English Justification | +| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `storage` | Saves the user's preferred Rate.vn server endpoint, a temporary product lookup cache for the active session, and the user's explicit opt-in preference for contributing product price signals. | +| `sidePanel` | Displays the price history chart, cross-merchant price comparison table and AI review authenticity breakdown beside the shopping page when the user clicks the extension badge or floating pill. | +| `https://shopee.vn/*`, `https://*.lazada.vn/*`, `https://tiki.vn/*`, `https://www.tiktok.com/*`, `https://shop.tiktok.com/*`, `https://cellphones.com.vn/*`, `https://*.thegioididong.com/*`, `https://fptshop.com.vn/*`, `https://hoanghamobile.com/*`, `https://didongviet.vn/*`, `https://phongvu.vn/*` | Runs the content script on the 10 supported Vietnamese stores. It reads the public product title, selected variant, listed price, original price and rating of a product page to show the floating price-check badge. The script loads on every page of these sites because they navigate in-page, but it reads and sends nothing on pages it does not recognise as a product page. TikTok is limited to the two hosts its product pages use (DOS/Rate#1232). | +| `https://rate.vn/*`, `https://www.rate.vn/*` | Fetches verified price history, cross-merchant offers and review authenticity scores from the Rate.vn API, and sends the opt-in crowd signal. | --- ## 3. Privacy & User Data Disclosure -1. **No Personal or Browsing History Profiling:** Rate.vn only activates on supported product detail pages (`shopee.vn`, `lazada.vn`, `tiki.vn`, `cellphones.com.vn`, `thegioididong.com`, `fptshop.com.vn`). -2. **Automatic URL Canonicalization (Tracking Stripping):** Before performing any product lookup, all query parameters and tracking tokens (`utm_*`, `sp_atk`, `xptdk`, `fbclid`, `gclid`) are stripped locally in the browser. Only the clean product pathname, product title, selected storage variant, and displayed price are used for lookup. -3. **Opt-In Crowdsourced Verification Queue:** By default, the extension operates in **100% Read-Only Lookup Mode** (`GET /api/v1/rate/extension/lookup`). Enqueuing newly discovered public product listings for server-side verification (`POST /api/v1/rate/extension/ingest`) is **disabled by default** and requires the user to explicitly enable the "Đóng góp phát hiện sản phẩm mới (Opt-In)" toggle in the Side Panel. +1. **Where it runs:** the content script loads on the 10 supported stores listed above (`shopee.vn`, `lazada.vn`, `tiki.vn`, `www.tiktok.com`, `shop.tiktok.com`, `cellphones.com.vn`, `thegioididong.com`, `fptshop.com.vn`, `hoanghamobile.com`, `didongviet.vn`, `phongvu.vn`). It acts only on pages it recognises as a product page; it never opens or fetches pages the user did not open. +2. **Lookup (on by default, no opt-in):** on a recognised product page the extension sends `GET /api/v1/rate/extension/lookup` with the store, the product URL with query parameters and tracking tokens (`utm_*`, `sp_atk`, `xptdk`, `fbclid`, `gclid`) stripped locally, the product title, the store's item id and shop id, the selected variant, the listed price and the original price. The request is read-only and carries no account or personal data. +3. **Opt-in crowd signal (off by default):** only when the user turns on "Đóng góp phát hiện sản phẩm mới (chỉ khi bạn bật)" in the Side Panel, the extension also sends `POST /api/v1/rate/extension/ingest` with the store, item id, shop id, cleaned URL, title, selected variant, price, original price, rating and rating count. The toggle's label lists these same fields (DOS/Rate#1230). Submissions go to a moderation queue and never change public prices directly. +4. **Stored in the browser only:** the server address and the opt-in choice (`chrome.storage.local`), and a session-only cache of the products viewed in each tab, the latest side panel search and their lookup results (`chrome.storage.session`). +5. **Side panel requests:** text the user types into the side panel search is sent to the same lookup endpoint; the side panel also reads public deals and merchant lists from the Rate.vn API. +6. **Privacy policy:** https://rate.vn/privacy-policy has a section on the extension. --- diff --git a/apps/extension/manifest.json b/apps/extension/manifest.json index 603f56165..47399927f 100644 --- a/apps/extension/manifest.json +++ b/apps/extension/manifest.json @@ -1,16 +1,17 @@ { "manifest_version": 3, - "name": "Rate.vn - So Sánh Giá, Lịch Sử Giá & Lọc Đánh Giá Ảo AI", + "name": "__MSG_extName__", "short_name": "Rate.vn", "version": "2.0.0", - "description": "Kiểm tra lịch sử giá 90 ngày, phát hiện giảm giá ảo, tìm nơi bán rẻ hơn và lọc đánh giá seeding bằng AI trên 10 sàn TMĐT & chuỗi bán lẻ Việt Nam.", + "description": "__MSG_extDescription__", + "default_locale": "vi", "icons": { "16": "icons/icon-16.png", "48": "icons/icon-48.png", "128": "icons/icon-128.png" }, "action": { - "default_title": "Mở bảng phân tích giá & độ tin cậy Rate.vn", + "default_title": "__MSG_actionTitle__", "default_icon": { "16": "icons/icon-16.png", "48": "icons/icon-48.png", @@ -24,15 +25,13 @@ "service_worker": "background/service-worker.js", "type": "module" }, - "permissions": [ - "storage", - "sidePanel" - ], + "permissions": ["storage", "sidePanel"], "host_permissions": [ "https://shopee.vn/*", "https://*.lazada.vn/*", "https://tiki.vn/*", - "https://*.tiktok.com/*", + "https://www.tiktok.com/*", + "https://shop.tiktok.com/*", "https://cellphones.com.vn/*", "https://*.thegioididong.com/*", "https://fptshop.com.vn/*", @@ -48,7 +47,8 @@ "https://shopee.vn/*", "https://*.lazada.vn/*", "https://tiki.vn/*", - "https://*.tiktok.com/*", + "https://www.tiktok.com/*", + "https://shop.tiktok.com/*", "https://cellphones.com.vn/*", "https://*.thegioididong.com/*", "https://fptshop.com.vn/*", diff --git a/apps/extension/scripts/build.mjs b/apps/extension/scripts/build.mjs index bd62303b0..f8b0892fd 100644 --- a/apps/extension/scripts/build.mjs +++ b/apps/extension/scripts/build.mjs @@ -31,7 +31,13 @@ async function main() { ); } - // 3. Bundle each entry with esbuild so every output is self-contained. + // 3. Copy the chrome.i18n catalogues (manifest default_locale "vi", + // DOS/Rate#1304); Chrome reads them from _locales next to manifest.json. + fs.cpSync(path.join(srcDir, '_locales'), path.join(distDir, '_locales'), { + recursive: true, + }); + + // 4. Bundle each entry with esbuild so every output is self-contained. // The service worker and side panel load as ES modules (manifest // "type": "module",