From e8797a09385f045fef91284183ddc042e5a32e7e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 03:58:05 +0000 Subject: [PATCH] feat(rest): preserve the original audit timeline for a historical import (#3493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #3479/#3483. `treatAsHistorical` skipped the state machine but the platform still rewrote the timeline: an imported row stamped `updated_at` / `updated_by` to the import instant, and an `upsert` refresh silently stripped business `readonly` fields (`closed_at`, `resolved_by`). Reports, audit, and "recently modified" sorting all came out wrong. Introduce an opt-in `ExecutionContext.preserveAudit` flag (server-set only) that `treatAsHistorical` sets alongside `skipStateMachine`, and make the three layers that force-overwrite the timeline respect it: - objectql audit hook (plugin.ts): `updated_at` / `updated_by` become client-preferred (`?? now` / `?? userId`) under preserveAudit, symmetric with how `created_at` / `created_by` already behave on insert. - objectql readonly strip (rule-validator.ts): admits a WHITELIST — the audit/timestamp family plus author-declared business `readonly` fields — while platform-managed `system` columns outside that family (`organization_id` / tenancy, generated columns) stay stripped. A whitelist, not the blanket `isSystem` exemption, so it is not a tenancy-forging backdoor. - driver-sql update: keeps a supplied `updated_at` instead of force-advancing it to `now` (`DriverOptions.preserveAudit`). Fully opt-in: a normal write still auto-stamps and strips exactly as before. Permissions / RLS / field-level security are unaffected. The objectui "Import as historical data" checkbox (objectui#2815) now drives both halves — no new UI. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01B5rdfBKjkbcoEif4KUV6xE --- .changeset/import-historical-audit.md | 44 ++++++++++++ content/docs/references/data/driver.mdx | 1 + .../references/kernel/execution-context.mdx | 1 + packages/objectql/src/engine.ts | 16 ++++- .../objectql/src/plugin.integration.test.ts | 69 +++++++++++++++++++ packages/objectql/src/plugin.ts | 10 ++- .../src/validation/rule-validator.test.ts | 66 ++++++++++++++++++ .../objectql/src/validation/rule-validator.ts | 46 +++++++++++++ .../src/sql-driver-timestamp-format.test.ts | 28 ++++++++ packages/plugins/driver-sql/src/sql-driver.ts | 16 ++++- .../rest/src/import-runner-historical.test.ts | 14 ++-- packages/rest/src/import-runner.ts | 20 ++++-- packages/spec/src/data/driver.zod.ts | 10 +++ .../spec/src/kernel/execution-context.zod.ts | 29 ++++++++ 14 files changed, 352 insertions(+), 18 deletions(-) create mode 100644 .changeset/import-historical-audit.md diff --git a/.changeset/import-historical-audit.md b/.changeset/import-historical-audit.md new file mode 100644 index 0000000000..6a70ca742c --- /dev/null +++ b/.changeset/import-historical-audit.md @@ -0,0 +1,44 @@ +--- +"@objectstack/spec": patch +"@objectstack/objectql": patch +"@objectstack/driver-sql": patch +"@objectstack/rest": patch +--- + +feat(rest): `treatAsHistorical` import also preserves the original audit timeline (#3493) + +Follow-up to #3479/#3483. `treatAsHistorical` solved the FSM half — mid-lifecycle +rows are no longer rejected by `initialStates` — but the OTHER half of a historical +migration, preserving the original timeline, still didn't hold: an imported ticket +that closed in 2021 stored `updated_at` = the import day (and `updated_by` = the +importer), and a `writeMode: 'upsert'` refresh silently dropped business `readonly` +fields (`closed_at`, `resolved_by`). Reports, audit, and "recently modified" +sorting all came out wrong. + +Three layers were force-overwriting the timeline; all three now respect a single +new opt-in flag, `ExecutionContext.preserveAudit`, which `treatAsHistorical` sets +alongside `skipStateMachine`: + +- **spec**: `ExecutionContext.preserveAudit` (server-set only, never client-supplied) + and `DriverOptions.preserveAudit` (threaded to the driver's update stamp). +- **objectql** — the built-in audit hook (`plugin.ts`) now treats `updated_at` / + `updated_by` as CLIENT-PREFERRED (`?? now` / `?? userId`) under `preserveAudit`, + symmetric with how `created_at` / `created_by` already behave on insert; and the + static-`readonly` write strip (`stripReadonlyFields`) admits a WHITELIST — the + audit/timestamp family plus author-declared business `readonly` fields — so an + upsert refresh no longer drops them. +- **driver-sql** — the SQL `update` path keeps a supplied `updated_at` instead of + force-advancing it to `now` when `DriverOptions.preserveAudit` is set (fills-only- + empty, mirroring the insert stamp). +- **rest** — the import runner sets `preserveAudit` on the write context iff the + request opts into `treatAsHistorical`. + +Deliberately a WHITELIST, not the blanket `isSystem` exemption: platform-managed +`system` columns OUTSIDE the audit family (`organization_id` / tenancy, generated +columns) STAY stripped, so a historical import reinstates established facts without +becoming a backdoor to forge tenancy. Permissions / RLS / field-level security are +unaffected — this changes only which audit/readonly values the runtime overwrites, +never who may write the record. Fully opt-in: a normal write still auto-stamps +`updated_at`/`updated_by` and strips `readonly` exactly as before. The objectui +"Import as historical data" checkbox (objectui#2815) now drives both halves — no new +UI. diff --git a/content/docs/references/data/driver.mdx b/content/docs/references/data/driver.mdx index 8bfdab665d..e26295497b 100644 --- a/content/docs/references/data/driver.mdx +++ b/content/docs/references/data/driver.mdx @@ -96,6 +96,7 @@ const result = DriverCapabilities.parse(data); | **traceContext** | `Record` | optional | OpenTelemetry context or request ID | | **tenantId** | `string` | optional | Tenant Isolation identifier | | **timezone** | `string` | optional | Business reference timezone (IANA) for date-dependent generation, e.g. autonumber date tokens | +| **preserveAudit** | `boolean` | optional | Historical import: keep a supplied updated_at instead of force-stamping now (from ExecutionContext.preserveAudit) | --- diff --git a/content/docs/references/kernel/execution-context.mdx b/content/docs/references/kernel/execution-context.mdx index c81b916a20..e43e89dd18 100644 --- a/content/docs/references/kernel/execution-context.mdx +++ b/content/docs/references/kernel/execution-context.mdx @@ -69,6 +69,7 @@ const result = ExecutionContext.parse(data); | **skipAutomations** | `boolean` | optional | | | **seedReplay** | `boolean` | optional | | | **skipStateMachine** | `boolean` | optional | | +| **preserveAudit** | `boolean` | optional | | | **oauthScopes** | `string[]` | optional | | | **accessToken** | `string` | optional | | | **transaction** | `any` | optional | | diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index b3d3a11bae..654e0a7f38 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -782,6 +782,10 @@ export class ObjectQL implements IDataEngine { // Propagate the full automation opt-out so `triggerHooks` can skip // metadata-bound hooks (import with "run automations" unchecked, undo). ...((execCtx as any).skipAutomations ? { skipAutomations: true } : {}), + // Propagate the historical-import audit-preservation flag so the built-in + // audit hook keeps a client-supplied updated_at/updated_by instead of + // stamping now (#3493). Opt-in, server-set only. + ...((execCtx as any).preserveAudit ? { preserveAudit: true } : {}), } as HookContext['session']; } @@ -858,7 +862,8 @@ export class ObjectQL implements IDataEngine { !isTenancyDisabled(this._registry.getObject(object)); const hasTz = execCtx?.timezone !== undefined; const isSystem = execCtx?.isSystem === true; - if (!hasTx && !hasTenant && !isSystem && !hasTz) return base; + const preserveAudit = (execCtx as any)?.preserveAudit === true; + if (!hasTx && !hasTenant && !isSystem && !hasTz && !preserveAudit) return base; const opts: any = base && typeof base === 'object' ? { ...base } : {}; if (hasTx && opts.transaction === undefined) { opts.transaction = tx; @@ -877,6 +882,11 @@ export class ObjectQL implements IDataEngine { // still flag genuine user-path bugs. opts.bypassTenantAudit = true; } + if (preserveAudit && opts.preserveAudit === undefined) { + // Historical import (#3493): let the driver keep a supplied `updated_at` + // instead of force-stamping now on the update path. + opts.preserveAudit = true; + } return opts; } @@ -2868,7 +2878,7 @@ export class ObjectQL implements IDataEngine { // read-only writes are dropped, never the server stamps. if (!opCtx.context?.isSystem) { const preRo = hookContext.input.data as Record; - hookContext.input.data = stripReadonlyFields(updateSchema as any, preRo, suppliedKeys, this.logger) as any; + hookContext.input.data = stripReadonlyFields(updateSchema as any, preRo, suppliedKeys, this.logger, { preserveAudit: opCtx.context?.preserveAudit === true }) as any; reportDroppedFields(preRo, hookContext.input.data as Record, 'readonly'); } evaluateValidationRules(updateSchema as any, hookContext.input.data as Record, 'update', { previous: priorRecord, logger: this.logger, currentUser: this.buildEvalUser(opCtx.context), skipStateMachine: shouldSkipStateMachine(opCtx.context) }); @@ -2923,7 +2933,7 @@ export class ObjectQL implements IDataEngine { // rejected upstream by the tenant write wall, #2946). if (!opCtx.context?.isSystem) { const preRoMulti = hookContext.input.data as Record; - hookContext.input.data = stripReadonlyFields(updateSchema as any, preRoMulti, suppliedKeys, this.logger) as any; + hookContext.input.data = stripReadonlyFields(updateSchema as any, preRoMulti, suppliedKeys, this.logger, { preserveAudit: opCtx.context?.preserveAudit === true }) as any; reportDroppedFields(preRoMulti, hookContext.input.data as Record, 'readonly'); } // [#3106] Same enforcement the single-id branch runs at its diff --git a/packages/objectql/src/plugin.integration.test.ts b/packages/objectql/src/plugin.integration.test.ts index 2a98d6e2ec..4484d0bd48 100644 --- a/packages/objectql/src/plugin.integration.test.ts +++ b/packages/objectql/src/plugin.integration.test.ts @@ -1380,6 +1380,75 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => { }); }); + // #3493 — an opt-in "historical" import (context.preserveAudit) reinstates the + // ORIGINAL timeline on UPDATE: the audit hook keeps a client-supplied + // updated_at/updated_by (instead of stamping now/importer), and the readonly + // strip admits the audit family + author-declared business readonly fields + // (closed_at) instead of dropping them. A normal write (no flag) still stamps + // and strips. (The driver's own updated_at force-stamp is covered separately in + // driver-sql's timestamp-format test — this mock driver echoes the data.) + describe('preserveAudit — historical import keeps the original timeline on UPDATE (#3493)', () => { + async function bootHistorical() { + const updates: Record[] = []; + const mockDriver = { + name: 'hist-capture', version: '1.0.0', + connect: async () => {}, disconnect: async () => {}, + find: async () => [], findOne: async () => null, + create: async (_o: string, d: any) => ({ id: 'rec-1', ...d }), + update: async (_o: string, _i: any, d: any) => { updates.push({ ...d }); return { id: _i, ...d }; }, + delete: async () => true, syncSchema: async () => {}, + }; + await kernel.use({ + name: 'hist-capture-plugin', type: 'driver', version: '1.0.0', + init: async (ctx) => { ctx.registerService('driver.hist-capture', mockDriver); }, + }); + await kernel.use(new ObjectQLPlugin()); + await kernel.bootstrap(); + const objectql = kernel.getService('objectql') as any; + const obj: ObjectSchema = { + name: 'ticket_obj', label: 'Ticket', datasource: 'hist-capture', + fields: { + name: { name: 'name', label: 'Name', type: 'text' }, + updated_by: { name: 'updated_by', label: 'Updated By', type: 'text', readonly: true } as any, + // author-declared business readonly field — a case's close time + closed_at: { name: 'closed_at', label: 'Closed At', type: 'datetime', readonly: true } as any, + }, + }; + objectql.registry.registerObject(obj, 'test', 'test'); + return { objectql, updates }; + } + + it('KEEPS a supplied updated_at / updated_by / closed_at under preserveAudit', async () => { + const { objectql, updates } = await bootHistorical(); + const ts = '2021-03-01T09:00:00.000Z'; + await objectql.update( + 'ticket_obj', + { id: 'rec-1', name: 'B', updated_at: ts, updated_by: 'u_original', closed_at: ts }, + { context: { userId: 'u_import', preserveAudit: true } }, + ); + expect(updates.length).toBe(1); + const data = updates[0]; + expect(data.name).toBe('B'); + expect(data.updated_at).toBe(ts); // hook kept client value (not "now") + expect(data.updated_by).toBe('u_original'); // hook kept client value (not importer) + expect(data.closed_at).toBe(ts); // business readonly reinstated, not stripped + }); + + it('a normal update (no preserveAudit) still overwrites updated_by and strips closed_at', async () => { + const { objectql, updates } = await bootHistorical(); + await objectql.update( + 'ticket_obj', + { id: 'rec-1', name: 'B', closed_at: '2021-03-01T09:00:00.000Z' }, + { context: { userId: 'u_import' } }, + ); + expect(updates.length).toBe(1); + const data = updates[0]; + expect(data.name).toBe('B'); + expect(data.updated_by).toBe('u_import'); // server-stamped to the actor + expect(data).not.toHaveProperty('closed_at'); // readonly business field stripped + }); + }); + // #3042 — conditional `readonlyWhen` must be enforced on the BULK // (updateMany) path too, not only the single-id path. The bulk strip reads // the matched rows' prior state and drops a field locked in ≥1 of them. diff --git a/packages/objectql/src/plugin.ts b/packages/objectql/src/plugin.ts index 1232b5a709..f14797b2ad 100644 --- a/packages/objectql/src/plugin.ts +++ b/packages/objectql/src/plugin.ts @@ -759,16 +759,22 @@ export class ObjectQLPlugin implements Plugin { isInsert: boolean, ) => { const now = stamp(); + // A "historical" import (#3493) reinstates the ORIGINAL timeline, so a + // client-supplied updated_at/updated_by is CLIENT-PREFERRED here — + // symmetric with created_at/created_by on insert — instead of being + // overwritten with the import instant. Opt-in and server-set only; a + // normal write leaves `preserveAudit` unset and still stamps now. + const preserveAudit = session?.preserveAudit === true; if (isInsert) { record.created_at = record.created_at ?? now; } - record.updated_at = now; + record.updated_at = preserveAudit ? (record.updated_at ?? now) : now; if (session?.userId) { if (isInsert && hasField(objectName, 'created_by')) { record.created_by = record.created_by ?? session.userId; } if (hasField(objectName, 'updated_by')) { - record.updated_by = session.userId; + record.updated_by = preserveAudit ? (record.updated_by ?? session.userId) : session.userId; } } // Stamp the driver-layer `tenant_id` column from the caller's active org. diff --git a/packages/objectql/src/validation/rule-validator.test.ts b/packages/objectql/src/validation/rule-validator.test.ts index d80f54e893..1e45d3d606 100644 --- a/packages/objectql/src/validation/rule-validator.test.ts +++ b/packages/objectql/src/validation/rule-validator.test.ts @@ -153,6 +153,72 @@ describe('stripReadonlyFields (#2948)', () => { }); }); +// #3493 — a "historical" import (preserveAudit) reinstates the original +// timeline: the audit/timestamp family and author-declared business `readonly` +// fields survive the strip, while platform-managed `system` columns outside the +// audit family stay stripped (no tenancy-forging backdoor). +const historicalFields = { + fields: { + title: { type: 'text' }, + created_at: { type: 'datetime', readonly: true, system: true }, + created_by: { type: 'lookup', readonly: true, system: true }, + updated_at: { type: 'datetime', readonly: true, system: true }, + updated_by: { type: 'lookup', readonly: true, system: true }, + organization_id: { type: 'lookup', readonly: true, system: true }, + // author-declared business readonly field (NOT system) — e.g. a case's close time + closed_at: { type: 'datetime', readonly: true }, + }, +}; + +describe('stripReadonlyFields — preserveAudit whitelist (#3493)', () => { + it('KEEPS the caller-supplied audit/timestamp family under preserveAudit', () => { + const supplied = new Set(['created_at', 'created_by', 'updated_at', 'updated_by']); + const data = { + created_at: '2020-01-01T00:00:00Z', + created_by: 'u_creator', + updated_at: '2021-03-01T00:00:00Z', + updated_by: 'u_old', + }; + const out = stripReadonlyFields(historicalFields, { ...data }, supplied, undefined, { preserveAudit: true }); + expect(out).toEqual(data); + }); + + it('KEEPS an author-declared business readonly field (closed_at) under preserveAudit', () => { + const supplied = new Set(['closed_at']); + const out = stripReadonlyFields( + historicalFields, + { closed_at: '2021-03-01T00:00:00Z' }, + supplied, + undefined, + { preserveAudit: true }, + ); + expect(out).toEqual({ closed_at: '2021-03-01T00:00:00Z' }); + }); + + it('STILL strips a non-audit system column (organization_id) under preserveAudit — no tenancy backdoor', () => { + const supplied = new Set(['organization_id', 'closed_at']); + const out = stripReadonlyFields( + historicalFields, + { organization_id: 'org_forged', closed_at: '2021-03-01T00:00:00Z' }, + supplied, + undefined, + { preserveAudit: true }, + ); + expect(out).toEqual({ closed_at: '2021-03-01T00:00:00Z' }); + }); + + it('strips the whole family as before when preserveAudit is NOT set (regression)', () => { + const supplied = new Set(['updated_at', 'closed_at', 'organization_id']); + const out = stripReadonlyFields(historicalFields, { + title: 'x', + updated_at: '2021-03-01T00:00:00Z', + closed_at: '2021-03-01T00:00:00Z', + organization_id: 'o1', + }, supplied); + expect(out).toEqual({ title: 'x' }); + }); +}); + describe('needsPriorRecord — field conditional rules (B2)', () => { it('is true when a field declares requiredWhen / readonlyWhen', () => { expect(needsPriorRecord(invoiceFields as any)).toBe(true); diff --git a/packages/objectql/src/validation/rule-validator.ts b/packages/objectql/src/validation/rule-validator.ts index 1e8d5e54d3..46974f90a8 100644 --- a/packages/objectql/src/validation/rule-validator.ts +++ b/packages/objectql/src/validation/rule-validator.ts @@ -325,6 +325,15 @@ export function stripReadonlyWhenFieldsMulti( * system-context writes (import, seed replay, approvals, lifecycle hooks — * all `isSystem: true`) legitimately set read-only columns and skip it. * + * `options.preserveAudit` (#3493) relaxes the strip for an opt-in "historical" + * import that reinstates the original timeline: a caller-supplied read-only + * field is KEPT when {@link isPreservableUnderAudit} allows it — the + * audit/timestamp family or any author-declared business `readonly` field. + * Platform-managed `system` columns outside that family (`organization_id` / + * tenancy, generated columns) stay stripped, so the relaxation reinstates facts + * without becoming a tenancy-forging backdoor. It is a WHITELIST, deliberately + * narrower than the blanket `isSystem` exemption above. + * * Returns the same object when nothing is stripped, else a shallow copy with the * offending keys removed. */ @@ -333,14 +342,17 @@ export function stripReadonlyFields( data: Record | undefined | null, suppliedKeys: ReadonlySet, logger?: EvaluateRulesOptions['logger'], + options?: { preserveAudit?: boolean }, ): Record | undefined | null { const fields = objectSchema?.fields; if (!fields || !data) return data; + const preserveAudit = options?.preserveAudit === true; let result = data; for (const [name, def] of Object.entries(fields)) { if (!def?.readonly) continue; if (!(name in (result as Record))) continue; if (!suppliedKeys.has(name)) continue; // server-stamped, not caller-supplied — keep + if (preserveAudit && isPreservableUnderAudit(name, def)) continue; // historical import reinstates it if (result === data) result = { ...data }; delete (result as Record)[name]; logger?.warn?.(`Field '${name}' is read-only — ignoring incoming change (#2948)`); @@ -348,6 +360,36 @@ export function stripReadonlyFields( return result; } +/** + * The audit / attribution family — the "original timeline" a historical import + * (`preserveAudit`) is allowed to reinstate even though these columns are + * `system` + `readonly`. Kept in sync with the registry's auto-injected audit + * fields (`packages/objectql/src/registry.ts`). + */ +const AUDIT_TIMELINE_FIELDS: ReadonlySet = new Set([ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', +]); + +/** + * Whether a caller-supplied `readonly` field may be REINSTATED by an opt-in + * historical import (`preserveAudit`), rather than stripped (#3493). Allows: + * - the audit/timestamp family (`created_*` / `updated_*`) — the timeline the + * feature exists to preserve; and + * - author-declared business `readonly` fields (`closed_at`, `resolved_by`, …), + * i.e. anything NOT flagged `system: true`. + * Still strips platform-managed `system` columns outside the audit family + * (`organization_id` / tenancy, generated columns): a historical import may + * reinstate established facts, but must not forge tenancy or system-generated + * values. + */ +function isPreservableUnderAudit(name: string, def: ConditionalFieldDef): boolean { + if (AUDIT_TIMELINE_FIELDS.has(name)) return true; + return def.system !== true; +} + /** * A rule needs the prior record if it reasons about the transition or compares * against unchanged fields (`state_machine` / `cross_field` / `script`), or if @@ -383,6 +425,10 @@ interface ConditionalFieldDef { readonlyWhen?: string | Expression; /** Static, unconditional read-only flag (`field.readonly`). #2948. */ readonly?: boolean; + /** Auto-injected platform/audit field (`field.system`). Distinguishes the + * audit family / tenancy columns from author-declared business fields when + * a historical import (`preserveAudit`) relaxes the readonly strip. #3493. */ + system?: boolean; /** Field type — scopes per-option `visibleWhen` enforcement to choice fields. */ type?: string; /** select/multiselect/radio/checkboxes options; an option may gate itself with `visibleWhen`. */ diff --git a/packages/plugins/driver-sql/src/sql-driver-timestamp-format.test.ts b/packages/plugins/driver-sql/src/sql-driver-timestamp-format.test.ts index bcf0eaf65c..e6ab2864fc 100644 --- a/packages/plugins/driver-sql/src/sql-driver-timestamp-format.test.ts +++ b/packages/plugins/driver-sql/src/sql-driver-timestamp-format.test.ts @@ -66,6 +66,34 @@ describe('SqlDriver canonical audit-timestamp format (SQLite)', () => { expect(updated.updated_at >= updated.created_at).toBe(true); }); + // ── #3493 — historical import preserves a supplied updated_at on UPDATE ────── + + it('update({ preserveAudit }) KEEPS a caller-supplied updated_at instead of force-stamping now', async () => { + await driver.create('thing', { id: 'h1', name: 'A' }, { bypassTenantAudit: true }); + const historical = '2021-03-01T09:00:00.000Z'; + await driver.update('thing', 'h1', { name: 'B', updated_at: historical }, { bypassTenantAudit: true, preserveAudit: true } as any); + const row = await raw('thing').where('id', 'h1').first(); + expect(row.name).toBe('B'); + expect(row.updated_at).toBe(historical); // original timeline preserved, not "now" + }); + + it('update({ preserveAudit }) with NO supplied updated_at still stamps now (fills-only-empty)', async () => { + await driver.create('thing', { id: 'h2', name: 'A' }, { bypassTenantAudit: true }); + await new Promise((r) => setTimeout(r, 5)); + await driver.update('thing', 'h2', { name: 'B' }, { bypassTenantAudit: true, preserveAudit: true } as any); + const row = await raw('thing').where('id', 'h2').first(); + expect(row.updated_at).toMatch(ISO_Z); + }); + + it('update() WITHOUT preserveAudit force-advances updated_at even if the caller supplies one (regression)', async () => { + await driver.create('thing', { id: 'h3', name: 'A' }, { bypassTenantAudit: true }); + const historical = '2021-03-01T09:00:00.000Z'; + await driver.update('thing', 'h3', { name: 'B', updated_at: historical }, { bypassTenantAudit: true }); + const row = await raw('thing').where('id', 'h3').first(); + expect(row.updated_at).toMatch(ISO_Z); + expect(row.updated_at).not.toBe(historical); // normal update overwrites with now + }); + it('no on-disk format mixing: an inserted-only row and an updated row share one format (SQL ORDER BY safe)', async () => { await driver.create('thing', { id: 'never', name: 'x' }, { bypassTenantAudit: true }); await driver.create('thing', { id: 'edited', name: 'y' }, { bypassTenantAudit: true }); diff --git a/packages/plugins/driver-sql/src/sql-driver.ts b/packages/plugins/driver-sql/src/sql-driver.ts index 0bc4b456ab..d1f8c4c2a4 100644 --- a/packages/plugins/driver-sql/src/sql-driver.ts +++ b/packages/plugins/driver-sql/src/sql-driver.ts @@ -1211,6 +1211,18 @@ export class SqlDriver implements IDataDriver { } } + /** + * True when the UPDATE path must KEEP a caller-supplied `updated_at` rather + * than force-advancing it to `now` (#3493). Only for an opt-in "historical" + * import (`DriverOptions.preserveAudit`, threaded from + * `ExecutionContext.preserveAudit`) that carries an explicit `updated_at` — + * mirroring how `stampInsertTimestamps` preserves an explicit value on insert. + * A normal update leaves the flag unset, so `updated_at` always advances. + */ + protected keepSuppliedUpdatedAt(formatted: Record, options?: DriverOptions): boolean { + return (options as any)?.preserveAudit === true && formatted.updated_at != null; + } + async update(object: string, id: string | number, data: Record, options?: DriverOptions): Promise { this.auditMissingTenant(object, 'update', options); const rotationShards = this.rotationShardsOf(object); @@ -1219,7 +1231,7 @@ export class SqlDriver implements IDataDriver { this.applyTenantScope(builder, object, options); const formatted = this.applyWriteColumnMap(object, this.formatInput(object, data)); - if (this.tablesWithTimestamps.has(object)) { + if (this.tablesWithTimestamps.has(object) && !this.keepSuppliedUpdatedAt(formatted, options)) { // Canonical instant format. On SQLite (no native timestamp type) stamp // full ISO-8601 WITH an explicit `Z` — matching the insert paths // (`stampInsertTimestamps`) so create and update agree on one @@ -1403,7 +1415,7 @@ export class SqlDriver implements IDataDriver { options?: DriverOptions, ): Promise { const formatted = this.applyWriteColumnMap(object, this.formatInput(object, data)); - if (this.tablesWithTimestamps.has(object)) { + if (this.tablesWithTimestamps.has(object) && !this.keepSuppliedUpdatedAt(formatted, options)) { formatted.updated_at = this.isSqlite ? new Date().toISOString() : this.knex.fn.now(); } for (const shard of shards) { diff --git a/packages/rest/src/import-runner-historical.test.ts b/packages/rest/src/import-runner-historical.test.ts index fa5673b722..b26512eec0 100644 --- a/packages/rest/src/import-runner-historical.test.ts +++ b/packages/rest/src/import-runner-historical.test.ts @@ -54,21 +54,27 @@ function makeProvider() { } describe('runImport — historical import skips the state machine (#3479)', () => { - it('sets skipStateMachine on the write context when treatAsHistorical is true', async () => { + it('sets skipStateMachine AND preserveAudit on the write context when treatAsHistorical is true (#3493)', async () => { const { p, contexts } = makeProvider(); const summary = await runImport({ ...baseOpts, p, rows: [{ name: 'a' }, { name: 'b' }], treatAsHistorical: true, }); expect(summary.created).toBe(2); expect(contexts.length).toBeGreaterThan(0); - for (const ctx of contexts) expect(ctx?.skipStateMachine).toBe(true); + for (const ctx of contexts) { + expect(ctx?.skipStateMachine).toBe(true); + expect(ctx?.preserveAudit).toBe(true); + } }); - it('does NOT set skipStateMachine for a normal import (the FSM still applies)', async () => { + it('does NOT set skipStateMachine / preserveAudit for a normal import (auto-stamp + FSM still apply)', async () => { const { p, contexts } = makeProvider(); await runImport({ ...baseOpts, p, rows: [{ name: 'a' }], treatAsHistorical: false }); expect(contexts.length).toBeGreaterThan(0); - for (const ctx of contexts) expect(ctx?.skipStateMachine).toBeUndefined(); + for (const ctx of contexts) { + expect(ctx?.skipStateMachine).toBeUndefined(); + expect(ctx?.preserveAudit).toBeUndefined(); + } }); it('defaults to off when the option is omitted (safe default — walk the FSM)', async () => { diff --git a/packages/rest/src/import-runner.ts b/packages/rest/src/import-runner.ts index 3239cf4338..608a38b237 100644 --- a/packages/rest/src/import-runner.ts +++ b/packages/rest/src/import-runner.ts @@ -102,9 +102,11 @@ export interface RunImportOptions { matchFields: string[]; dryRun: boolean; runAutomations: boolean; - /** #3479 — treat rows as established historical facts: the write context - * carries `skipStateMachine`, so mid-lifecycle values aren't rejected by the - * object's `state_machine` `initialStates` (insert) / `transitions` (update). + /** #3479 / #3493 — treat rows as established historical facts. The write + * context carries `skipStateMachine` (mid-lifecycle values aren't rejected by + * the object's `state_machine` `initialStates`/`transitions`) AND + * `preserveAudit` (a supplied `updated_at`/`updated_by` and audit/business + * `readonly` fields are preserved rather than stamped-now / stripped). * Optional here (runner default is off); `prepareImportRequest` always sets it. */ treatAsHistorical?: boolean; trimWhitespace: boolean; @@ -270,10 +272,14 @@ export function runImport(opts: RunImportOptions): Promise { const writeCtx = { ...(context ?? {}), skipAutomations: !runAutomations, - // #3479 — a "historical" import carries curated established facts, so the - // engine skips the state_machine rule for these writes (initialStates on - // insert, transitions on update). Default off: a normal import walks the FSM. - ...(treatAsHistorical ? { skipStateMachine: true } : {}), + // #3479 / #3493 — a "historical" import carries curated established facts: + // - skipStateMachine: the engine skips the state_machine rule (initialStates + // on insert, transitions on update) so mid-lifecycle rows aren't rejected; + // - preserveAudit: the ORIGINAL timeline is kept — a supplied + // updated_at/updated_by survives (not stamped now), and the audit/business + // readonly fields survive the upsert-update readonly strip. + // Default off: a normal import walks the FSM and auto-stamps as usual. + ...(treatAsHistorical ? { skipStateMachine: true, preserveAudit: true } : {}), }; // Sparse-indexed by row position `i` (not push-only): CREATE rows are diff --git a/packages/spec/src/data/driver.zod.ts b/packages/spec/src/data/driver.zod.ts index 7d27cccdd0..8fdabe9983 100644 --- a/packages/spec/src/data/driver.zod.ts +++ b/packages/spec/src/data/driver.zod.ts @@ -45,6 +45,16 @@ export const DriverOptionsSchema = lazySchema(() => z.object({ * (`{YYYYMMDD}`) — resolve the calendar day in this zone, falling back to UTC. */ timezone: z.string().optional().describe('Business reference timezone (IANA) for date-dependent generation, e.g. autonumber date tokens'), + + /** + * Preserve a caller-supplied `updated_at` on the UPDATE path instead of + * force-advancing it to `now` (#3493). Threaded from + * `ExecutionContext.preserveAudit` — set only for an opt-in "historical" data + * import that reinstates the original timeline. When true AND the row carries + * an explicit `updated_at`, the driver keeps it; otherwise it stamps `now` as + * usual. A normal update leaves this unset, so `updated_at` always advances. + */ + preserveAudit: z.boolean().optional().describe('Historical import: keep a supplied updated_at instead of force-stamping now (from ExecutionContext.preserveAudit)'), })); /** diff --git a/packages/spec/src/kernel/execution-context.zod.ts b/packages/spec/src/kernel/execution-context.zod.ts index 43b4b491b7..d6e970d0df 100644 --- a/packages/spec/src/kernel/execution-context.zod.ts +++ b/packages/spec/src/kernel/execution-context.zod.ts @@ -237,6 +237,35 @@ export const ExecutionContextSchema = lazySchema(() => z.object({ */ skipStateMachine: z.boolean().optional(), + /** + * Preserve the ORIGINAL audit timeline for this write instead of stamping it + * "now" (#3493, the other half of a "historical" import — {@link skipStateMachine} + * covers the FSM half). When set: + * + * - the built-in audit hook (`packages/objectql/src/plugin.ts`) treats + * `updated_at` / `updated_by` as CLIENT-PREFERRED (`?? now` / `?? userId`), + * symmetric with how `created_at` / `created_by` already behave on insert, + * so a supplied historical last-modified survives instead of being + * overwritten with the import instant; + * - the static-`readonly` write strip (`stripReadonlyFields`) admits a + * WHITELIST — the audit/timestamp family (`created_at` / `created_by` / + * `updated_at` / `updated_by`) plus author-declared business `readonly` + * fields (`closed_at`, `resolved_by`, …) — so migrating established facts + * doesn't silently drop them on the upsert-update path. Platform-managed + * `system` columns OUTSIDE that family (`organization_id` / tenancy, and + * any generated column) STAY stripped: this reinstates a timeline, it is + * NOT a backdoor to forge tenancy; + * - the SQL driver's update stamp respects a supplied `updated_at` rather + * than force-advancing it to `now` (`DriverOptions.preserveAudit`). + * + * Opt-IN and server-constructed only, never client-supplied — the REST import + * runner sets it from the request's explicit `treatAsHistorical` option, gated + * so a NORMAL write still auto-stamps and strips as before. Permissions / RLS / + * field-level security are unaffected: this changes only which audit/readonly + * values the runtime overwrites, never who may write the record. + */ + preserveAudit: z.boolean().optional(), + /** * OAuth 2.1 scopes granted to the access token that authenticated this * request, when the principal was resolved from an OAuth bearer token