From 2e645f2049d2ae0264827b157b777c521c11e36c Mon Sep 17 00:00:00 2001 From: Kevin McGahey <36458555+thekevinm@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:36:18 +0000 Subject: [PATCH] feat: Data Masking and Compliance screens (commercial) Admin UI for the df-masking package (GOLD): - Security > Data Masking: first run and scan progress, columns review with evidence, masks, detection rules, real-value access with expiry, activity, and the Off / Preview / On mode switch. - Security > Compliance: HIPAA, PCI DSS, GLBA, CJIS, GDPR, SOC 2 and ISO 27001 baselines with Apply, live checklist and evidence export. - Data Explorer notes when columns are masked or a filter/sort on a masked column was dropped; Security overview card; command palette keywords (pii, phi, pci, mask). - Both pages show the standard paywall whenever the df-masking resources are missing (open source, Silver, or Gold without the package), and are locked in the menu on open source and Silver. Unit specs plus opt-in Playwright suites (e2e/_data-masking-*). dist is not rebuilt. Claude-Session: https://claude.ai/code/session_01D3YUvzLr35XYve3pZwT3SW --- .prettierignore | 3 + e2e/_data-masking-live-audit.config.ts | 28 + e2e/_data-masking-live-audit.spec.ts | 1447 +++++++++++++++++ e2e/_data-masking-live.config.ts | 27 + e2e/_data-masking-live.spec.ts | 742 +++++++++ e2e/_data-masking-r2.spec.ts | 713 ++++++++ e2e/_data-masking-r3.spec.ts | 375 +++++ e2e/_data-masking-r4.spec.ts | 110 ++ e2e/_data-masking-screens.config.ts | 16 + e2e/_data-masking-screens.spec.ts | 754 +++++++++ e2e/fixtures/compliance-mock.ts | 731 +++++++++ e2e/fixtures/data-masking-mock.ts | 1268 +++++++++++++++ e2e/fixtures/data-masking/fixture.json | 1 + e2e/fixtures/data-masking/generate-fixture.ts | 438 +++++ jest.config.ci.js | 17 + .../compliance-api.service.spec.ts | 96 ++ .../adf-compliance/compliance-api.service.ts | 86 + src/app/adf-compliance/compliance-testing.ts | 115 ++ src/app/adf-compliance/compliance.catalog.ts | 45 + src/app/adf-compliance/compliance.resolver.ts | 14 + .../adf-compliance/compliance.store.spec.ts | 572 +++++++ src/app/adf-compliance/compliance.store.ts | 462 ++++++ src/app/adf-compliance/compliance.types.ts | 72 + ...-compliance-apply-dialog.component.spec.ts | 151 ++ .../df-compliance-apply-dialog.component.ts | 175 ++ .../df-compliance.component.html | 313 ++++ .../df-compliance.component.scss | 527 ++++++ .../df-compliance.component.spec.ts | 398 +++++ .../df-compliance/df-compliance.component.ts | 283 ++++ .../df-data-grid.component.ts | 122 ++ .../df-data-grid.masking.spec.ts | 251 +++ .../adf-data-explorer/masking-notice.spec.ts | 102 ++ src/app/adf-data-explorer/masking-notice.ts | 122 ++ .../services/data-explorer.service.ts | 30 +- src/app/adf-data-masking/_dm-controls.scss | 86 + src/app/adf-data-masking/_dm-tier.scss | 44 + .../data-masking-api.service.spec.ts | 154 ++ .../data-masking-api.service.ts | 201 +++ .../adf-data-masking/data-masking.catalog.ts | 274 ++++ .../data-masking.forbidden.spec.ts | 251 +++ .../adf-data-masking/data-masking.resolver.ts | 14 + .../data-masking.store.spec.ts | 1077 ++++++++++++ .../adf-data-masking/data-masking.store.ts | 1296 +++++++++++++++ .../adf-data-masking/data-masking.types.ts | 257 +++ .../df-class-picker.component.html | 67 + .../df-class-picker.component.scss | 170 ++ .../df-class-picker.component.ts | 67 + .../df-data-masking.component.html | 130 ++ .../df-data-masking.component.scss | 142 ++ .../df-data-masking.component.spec.ts | 438 +++++ .../df-data-masking.component.ts | 270 +++ .../df-dm-confirm-dialog.component.ts | 77 + .../df-dm-access/df-dm-access.component.html | 189 +++ .../df-dm-access/df-dm-access.component.scss | 196 +++ .../df-dm-access.component.spec.ts | 293 ++++ .../df-dm-access/df-dm-access.component.ts | 426 +++++ .../df-dm-activity.component.html | 71 + .../df-dm-activity.component.scss | 152 ++ .../df-dm-activity.component.spec.ts | 98 ++ .../df-dm-activity.component.ts | 152 ++ .../df-dm-columns/_dm-columns-first-run.scss | 165 ++ .../df-dm-columns/_dm-columns-rail.scss | 217 +++ .../df-dm-columns/_dm-columns-toolbar.scss | 184 +++ .../df-dm-columns.component.html | 592 +++++++ .../df-dm-columns.component.scss | 458 ++++++ .../df-dm-columns.component.spec.ts | 367 +++++ .../df-dm-columns/df-dm-columns.component.ts | 714 ++++++++ .../df-dm-columns.first-run.spec.ts | 452 +++++ .../df-dm-detection.component.html | 144 ++ .../df-dm-detection.component.scss | 240 +++ .../df-dm-detection.component.spec.ts | 266 +++ .../df-dm-detection.component.ts | 131 ++ .../df-dm-rule-dialog.component.html | 110 ++ .../df-dm-rule-dialog.component.scss | 124 ++ .../df-dm-rule-dialog.component.ts | 186 +++ .../df-dm-masks/df-dm-masks.component.html | 75 + .../df-dm-masks/df-dm-masks.component.scss | 112 ++ .../df-dm-masks/df-dm-masks.component.ts | 81 + .../df-dm-undo-toast.component.ts | 71 + .../df-evidence-ledger.component.html | 66 + .../df-evidence-ledger.component.scss | 190 +++ .../df-evidence-ledger.component.spec.ts | 46 + .../df-evidence-ledger.component.ts | 89 + .../df-masked-value.component.scss | 83 + .../df-masked-value.component.spec.ts | 155 ++ .../df-masked-value.component.ts | 162 ++ .../df-sensitivity-strip.component.scss | 43 + .../df-sensitivity-strip.component.spec.ts | 44 + .../df-sensitivity-strip.component.ts | 149 ++ src/app/adf-data-masking/dm-testing.ts | 39 + src/app/adf-data-masking/dm-time.spec.ts | 23 + src/app/adf-data-masking/dm-time.ts | 39 + src/app/adf-data-masking/mask-format.spec.ts | 71 + src/app/adf-data-masking/mask-format.ts | 178 ++ .../df-security-overview.component.spec.ts | 128 ++ .../df-security-overview.component.ts | 90 +- src/app/routes.ts | 20 + .../df-section-landing.component.html | 5 +- .../df-section-landing.component.scss | 10 + .../df-section-landing.component.ts | 6 + .../services/action-registry.service.spec.ts | 133 ++ .../services/action-registry.service.ts | 69 +- .../services/df-paywall.service.spec.ts | 98 ++ src/app/shared/services/df-paywall.service.ts | 10 +- src/app/shared/types/routes.ts | 2 + src/assets/i18n/compliance/en.json | 55 + src/assets/i18n/dataExplorer/en.json | 8 +- src/assets/i18n/dataMasking/en.json | 279 ++++ src/assets/i18n/en.json | 21 + src/styles.scss | 43 + 110 files changed, 24218 insertions(+), 23 deletions(-) create mode 100644 e2e/_data-masking-live-audit.config.ts create mode 100644 e2e/_data-masking-live-audit.spec.ts create mode 100644 e2e/_data-masking-live.config.ts create mode 100644 e2e/_data-masking-live.spec.ts create mode 100644 e2e/_data-masking-r2.spec.ts create mode 100644 e2e/_data-masking-r3.spec.ts create mode 100644 e2e/_data-masking-r4.spec.ts create mode 100644 e2e/_data-masking-screens.config.ts create mode 100644 e2e/_data-masking-screens.spec.ts create mode 100644 e2e/fixtures/compliance-mock.ts create mode 100644 e2e/fixtures/data-masking-mock.ts create mode 100644 e2e/fixtures/data-masking/fixture.json create mode 100644 e2e/fixtures/data-masking/generate-fixture.ts create mode 100644 src/app/adf-compliance/compliance-api.service.spec.ts create mode 100644 src/app/adf-compliance/compliance-api.service.ts create mode 100644 src/app/adf-compliance/compliance-testing.ts create mode 100644 src/app/adf-compliance/compliance.catalog.ts create mode 100644 src/app/adf-compliance/compliance.resolver.ts create mode 100644 src/app/adf-compliance/compliance.store.spec.ts create mode 100644 src/app/adf-compliance/compliance.store.ts create mode 100644 src/app/adf-compliance/compliance.types.ts create mode 100644 src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.spec.ts create mode 100644 src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.ts create mode 100644 src/app/adf-compliance/df-compliance/df-compliance.component.html create mode 100644 src/app/adf-compliance/df-compliance/df-compliance.component.scss create mode 100644 src/app/adf-compliance/df-compliance/df-compliance.component.spec.ts create mode 100644 src/app/adf-compliance/df-compliance/df-compliance.component.ts create mode 100644 src/app/adf-data-explorer/df-data-grid.masking.spec.ts create mode 100644 src/app/adf-data-explorer/masking-notice.spec.ts create mode 100644 src/app/adf-data-explorer/masking-notice.ts create mode 100644 src/app/adf-data-masking/_dm-controls.scss create mode 100644 src/app/adf-data-masking/_dm-tier.scss create mode 100644 src/app/adf-data-masking/data-masking-api.service.spec.ts create mode 100644 src/app/adf-data-masking/data-masking-api.service.ts create mode 100644 src/app/adf-data-masking/data-masking.catalog.ts create mode 100644 src/app/adf-data-masking/data-masking.forbidden.spec.ts create mode 100644 src/app/adf-data-masking/data-masking.resolver.ts create mode 100644 src/app/adf-data-masking/data-masking.store.spec.ts create mode 100644 src/app/adf-data-masking/data-masking.store.ts create mode 100644 src/app/adf-data-masking/data-masking.types.ts create mode 100644 src/app/adf-data-masking/df-class-picker/df-class-picker.component.html create mode 100644 src/app/adf-data-masking/df-class-picker/df-class-picker.component.scss create mode 100644 src/app/adf-data-masking/df-class-picker/df-class-picker.component.ts create mode 100644 src/app/adf-data-masking/df-data-masking/df-data-masking.component.html create mode 100644 src/app/adf-data-masking/df-data-masking/df-data-masking.component.scss create mode 100644 src/app/adf-data-masking/df-data-masking/df-data-masking.component.spec.ts create mode 100644 src/app/adf-data-masking/df-data-masking/df-data-masking.component.ts create mode 100644 src/app/adf-data-masking/df-data-masking/df-dm-confirm-dialog.component.ts create mode 100644 src/app/adf-data-masking/df-dm-access/df-dm-access.component.html create mode 100644 src/app/adf-data-masking/df-dm-access/df-dm-access.component.scss create mode 100644 src/app/adf-data-masking/df-dm-access/df-dm-access.component.spec.ts create mode 100644 src/app/adf-data-masking/df-dm-access/df-dm-access.component.ts create mode 100644 src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.html create mode 100644 src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.scss create mode 100644 src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.spec.ts create mode 100644 src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.ts create mode 100644 src/app/adf-data-masking/df-dm-columns/_dm-columns-first-run.scss create mode 100644 src/app/adf-data-masking/df-dm-columns/_dm-columns-rail.scss create mode 100644 src/app/adf-data-masking/df-dm-columns/_dm-columns-toolbar.scss create mode 100644 src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.html create mode 100644 src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.scss create mode 100644 src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.spec.ts create mode 100644 src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.ts create mode 100644 src/app/adf-data-masking/df-dm-columns/df-dm-columns.first-run.spec.ts create mode 100644 src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.html create mode 100644 src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.scss create mode 100644 src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.spec.ts create mode 100644 src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.ts create mode 100644 src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.html create mode 100644 src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.scss create mode 100644 src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.ts create mode 100644 src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.html create mode 100644 src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.scss create mode 100644 src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.ts create mode 100644 src/app/adf-data-masking/df-dm-undo-toast/df-dm-undo-toast.component.ts create mode 100644 src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.html create mode 100644 src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.scss create mode 100644 src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.spec.ts create mode 100644 src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.ts create mode 100644 src/app/adf-data-masking/df-masked-value/df-masked-value.component.scss create mode 100644 src/app/adf-data-masking/df-masked-value/df-masked-value.component.spec.ts create mode 100644 src/app/adf-data-masking/df-masked-value/df-masked-value.component.ts create mode 100644 src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.scss create mode 100644 src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.spec.ts create mode 100644 src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.ts create mode 100644 src/app/adf-data-masking/dm-testing.ts create mode 100644 src/app/adf-data-masking/dm-time.spec.ts create mode 100644 src/app/adf-data-masking/dm-time.ts create mode 100644 src/app/adf-data-masking/mask-format.spec.ts create mode 100644 src/app/adf-data-masking/mask-format.ts create mode 100644 src/app/adf-section-overviews/df-security-overview.component.spec.ts create mode 100644 src/app/shared/services/action-registry.service.spec.ts create mode 100644 src/app/shared/services/df-paywall.service.spec.ts create mode 100644 src/assets/i18n/compliance/en.json create mode 100644 src/assets/i18n/dataMasking/en.json diff --git a/.prettierignore b/.prettierignore index 8c6e3bd40..8d39927df 100644 --- a/.prettierignore +++ b/.prettierignore @@ -43,3 +43,6 @@ Thumbs.db # assets /src/assets/* + +# Generated by e2e/fixtures/data-masking/generate-fixture.ts (compact JSON) +/e2e/fixtures/data-masking/fixture.json diff --git a/e2e/_data-masking-live-audit.config.ts b/e2e/_data-masking-live-audit.config.ts new file mode 100644 index 000000000..305883b78 --- /dev/null +++ b/e2e/_data-masking-live-audit.config.ts @@ -0,0 +1,28 @@ +/** + * Runs only the Data Masking + Compliance live audit (real df-masking + * backend, no page.route mocks): every tab is compared with the system + * API, and the real-backend screenshots are saved. + * + * # ng serve proxied to a GOLD instance with df-masking installed + * PLAYWRIGHT_BASE_URL=http://127.0.0.1:4301 \ + * DM_LIVE_SCREENS_DIR=/path/to/screens/live \ + * npx playwright test -c e2e/_data-masking-live-audit.config.ts + * + * Optional: DM_OSS_BASE_URL (an admin UI proxied to an open-source + * DreamFactory) runs the paywall check; DM_OSS_EMAIL / DM_OSS_PASSWORD + * sign in there. + */ +import { defineConfig } from '@playwright/test'; +import base from '../playwright.config'; + +export default defineConfig({ + ...base, + testDir: '.', + testIgnore: [], + testMatch: ['_data-masking-live-audit.spec.ts'], + // A never-scanned instance scans every covered database first. + timeout: 20 * 60_000, + expect: { timeout: 20_000 }, + retries: 0, + workers: 1, +}); diff --git a/e2e/_data-masking-live-audit.spec.ts b/e2e/_data-masking-live-audit.spec.ts new file mode 100644 index 000000000..419d288b1 --- /dev/null +++ b/e2e/_data-masking-live-audit.spec.ts @@ -0,0 +1,1447 @@ +/** + * Data Masking + Compliance live audit against a REAL df-masking backend + * (no page.route mocks). Where _data-masking-live.spec.ts walks the + * journey, this one reads every tab and compares what is on screen with + * what the system API says: counts, rail, rows, examples, evidence, access, + * masks, rules (live test), activity, compliance statuses, Apply and + * Export. With DM_LIVE_SCREENS_DIR set it saves the real-backend + * screenshots (01-first-run ... 15-oss-paywall). + * + * Shared-stack discipline: everything it changes it puts back (mode, + * admin access, baseline tracking), except what the product makes sticky + * on purpose: Apply's tightening (detection rules a baseline adds) stays. + * Set DM_AUDIT_APPLY=0 to skip applying a baseline. + * + * PLAYWRIGHT_BASE_URL=http://127.0.0.1:4301 \ + * DM_LIVE_SCREENS_DIR=/path/to/screens/live \ + * DM_OSS_BASE_URL=http://127.0.0.1:4302 \ + * npx playwright test -c e2e/_data-masking-live-audit.config.ts + */ +import { + APIRequestContext, + Browser, + Locator, + Page, + expect, + test, +} from '@playwright/test'; +import { mkdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { ADMIN_EMAIL, ADMIN_PASSWORD } from './fixtures/admin-login'; + +const EMAIL = ADMIN_EMAIL; +const PASSWORD = ADMIN_PASSWORD; +const SCREENS = process.env['DM_LIVE_SCREENS_DIR'] ?? ''; +const OSS_URL = process.env['DM_OSS_BASE_URL'] ?? ''; +const OSS_EMAIL = process.env['DM_OSS_EMAIL'] ?? ADMIN_EMAIL; +const OSS_PASSWORD = process.env['DM_OSS_PASSWORD'] ?? ADMIN_PASSWORD; +/** An app (role without real-value access) and a covered table it reads. */ +const PREVIEW_KEY = process.env['DM_APP_KEY'] ?? ''; +const PREVIEW_PATH = + process.env['DM_APP_PATH'] ?? + '/api/v2/crm_mysql/_table/customers?limit=1&fields=id,email'; +/** A second admin who is not the super admin (tightens, never loosens). */ +const OTHER_EMAIL = process.env['DM_OTHER_EMAIL'] ?? ''; +const OTHER_PASSWORD = process.env['DM_OTHER_PASSWORD'] ?? ''; +const APPLY = process.env['DM_AUDIT_APPLY'] !== '0'; +const APPLY_BASELINE = process.env['DM_AUDIT_BASELINE'] ?? 'hipaa'; +/** On a never-scanned instance: the shortcut whose Apply starts the scan. */ +const FIRST_APPLY_BASELINE = process.env['DM_AUDIT_FIRST_BASELINE'] ?? 'soc2'; +const SCAN_TIMEOUT = 15 * 60_000; + +const DM = '/dreamfactory/dist/#/api-security/data-masking'; +const CP = '/dreamfactory/dist/#/api-security/compliance'; +const GLYPH = '\u2022'; +const TOKEN_RE = /^[A-Z][A-Z0-9]*_[0-9a-f]{4,16}$/; + +if (SCREENS) mkdirSync(SCREENS, { recursive: true }); + +const i18n = (scope: string) => + JSON.parse( + readFileSync(join(__dirname, `../src/assets/i18n/${scope}/en.json`), 'utf8') + ); +const DMT = i18n('dataMasking'); +const CPT = i18n('compliance'); + +/** Minimal system API client (raw wire format: snake_case). */ +class Api { + constructor( + private ctx: APIRequestContext, + private token: string + ) {} + + static async login(ctx: APIRequestContext): Promise { + const res = await ctx.post('/api/v2/system/admin/session', { + data: { email: EMAIL, password: PASSWORD }, + }); + expect(res.ok(), `admin API login: HTTP ${res.status()}`).toBe(true); + return new Api(ctx, (await res.json()).session_token); + } + + private h() { + return { 'X-DreamFactory-Session-Token': this.token }; + } + + async get(path: string): Promise { + const res = await this.ctx.get(`/api/v2${path}`, { headers: this.h() }); + expect(res.ok(), `GET ${path}: HTTP ${res.status()}`).toBe(true); + return res.json(); + } + + async send( + method: 'patch' | 'post' | 'delete', + path: string, + data?: unknown + ): Promise { + const res = await this.ctx[method](`/api/v2${path}`, { + headers: this.h(), + data, + }); + expect(res.ok(), `${method} ${path}: HTTP ${res.status()}`).toBe(true); + return res.json().catch(() => ({})); + } +} + +let page: Page; +let api: Api; +const featureErrors: string[] = []; +const pageErrors: string[] = []; +const snap: { + mode?: string; + admins?: { mode: string; grants: { admin_id: number }[] }; + ai?: string; + adopted?: string[]; +} = {}; + +/** Everything the Columns tab is built from. */ +interface Model { + overview: any; + services: any[]; + columns: Map; +} + +async function model(): Promise { + const [overview, tables] = await Promise.all([ + api.get('/system/data_masking'), + api.get('/system/data_masking/table'), + ]); + const columns = new Map(); + await Promise.all( + tables.resource.map(async (s: any) => { + const r = await api.get( + `/system/data_masking/column?service=${encodeURIComponent(s.service)}` + ); + columns.set(s.service, r.resource); + }) + ); + return { overview, services: tables.resource, columns }; +} + +const tableCols = (m: Model, svc: string, table: string) => + (m.columns.get(svc) ?? []).filter(c => c.table === table); + +async function hideChrome() { + // Third-party launcher and the test image's licence banner (its key + // cannot reach the licence server): neither is part of the feature. + await page.addStyleTag({ + content: + '[id^="intercom"], .intercom-lightweight-app, iframe[name^="intercom"], .license-expired { display: none !important; }', + }); +} + +async function shot( + name: string, + opts: { height?: number; top?: boolean } = {} +) { + if (!SCREENS) return; + await hideChrome(); + const size = page.viewportSize(); + if (opts.height && size) { + await page.setViewportSize({ width: size.width, height: opts.height }); + } + if (opts.top) { + await page.evaluate(() => { + document + .querySelectorAll('mat-sidenav-content, .mat-drawer-content') + .forEach(e => (e.scrollTop = 0)); + window.scrollTo(0, 0); + }); + } + await page.mouse.move(2, 2); + // No stray focus ring from the last click or key press. + await page.evaluate(() => (document.activeElement as HTMLElement)?.blur()); + await page.waitForTimeout(500); + await page.screenshot({ path: join(SCREENS, `${name}.png`), caret: 'hide' }); + if (opts.height && size) await page.setViewportSize(size); +} + +async function dismissToasts() { + await page + .locator('mat-snack-bar-container') + .evaluateAll(els => + els.forEach(e => ((e as HTMLElement).style.display = 'none')) + ); +} + +async function waitFor( + read: () => Promise, + ok: (v: T) => boolean, + what: string, + timeout = 20_000 +): Promise { + let last: T | undefined; + await expect + .poll( + async () => { + last = await read(); + return ok(last); + }, + { timeout, message: what } + ) + .toBe(true); + return last as T; +} + +async function openDataMasking(tab = '') { + await page.goto(`${DM}${tab ? `?tab=${tab}` : ''}`); + // Before the first scan with masking off the mode control is not drawn + // (the page leads with the scan), so wait for whichever view loads. + await expect( + page + .getByTestId('dm-mode') + .or(page.getByTestId('dm-no-scan')) + .or(page.getByTestId('dm-first-scan')) + .first() + ).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId('dm-tab-columns')).toHaveText(/\S/); +} + +async function login(p: Page, email: string, password: string) { + await p.context().clearCookies(); + await p.goto('/dreamfactory/dist/#/auth/login'); + const box = p.locator('input').first(); + await expect(box).toBeVisible({ timeout: 30_000 }); + await box.fill(email); + await p.locator('input').nth(1).fill(password); + const res = p.waitForResponse( + r => + r.url().includes('/api/v2/system/admin/session') && + r.request().method() === 'POST' + ); + await p.getByRole('button', { name: /^login$/i }).click(); + expect((await res).ok()).toBe(true); + await p.waitForURL(u => !u.toString().includes('/auth/login'), { + timeout: 30_000, + }); +} + +/** Rail groups as drawn: [{ label, tables: [{ name, badge }] }]. */ +async function railOnScreen() { + return page + .locator( + '.rail .rail-group:not(.is-pending):not(.is-uncovered):not(.is-quiet):not(.is-failed)' + ) + .evaluateAll(groups => + groups.map(g => ({ + label: (g.querySelector('.rail-service')?.textContent ?? '').trim(), + tables: Array.from( + g.querySelectorAll('[data-testid="dm-rail-row"]') + ).map(r => ({ + name: (r.querySelector('.rail-name')?.textContent ?? '').trim(), + badge: (r.querySelector('.badge')?.textContent ?? '').trim(), + strip: r + .querySelector('df-sensitivity-strip .strip') + ?.getAttribute('aria-label'), + })), + })) + ); +} + +/** Scanned databases listed quietly under the rail (nothing to list). */ +async function quietOnScreen() { + return page.locator('[data-testid="dm-rail-quiet-row"]').evaluateAll(rows => + rows.map(r => ({ + label: (r.querySelector('.rail-name')?.textContent ?? '').trim(), + note: (r.querySelector('.rail-note')?.textContent ?? '').trim(), + })) + ); +} + +function expectedQuiet(m: Model, filter: string) { + const inRail = new Set( + expectedRail( + m, + filter === 'review' + ? (x: any) => x.verdict === 'review' + : filter === 'masked' + ? (x: any) => x.verdict !== 'clear' + : () => true + ).map(g => g.label) + ); + return m.services + .filter(s => !inRail.has(s.label) && !s.scan_error) + .flatMap(s => { + if (!s.tables.length) { + return s.scanned_at + ? [{ label: s.label, note: DMT.columns.noTables }] + : []; + } + const sensitive = s.tables.some( + (t: any) => t.columns.masked + t.columns.review > 0 + ); + return !sensitive && (filter === 'masked' || filter === 'review') + ? [{ label: s.label, note: DMT.columns.nothingSensitive }] + : []; + }); +} + +function expectedRail(m: Model, keep: (c: any) => boolean) { + return m.services + .map(s => ({ + label: s.label, + tables: s.tables + .filter((t: any) => tableCols(m, s.service, t.name).some(keep)) + .map((t: any) => ({ + name: t.name, + badge: t.columns.review ? String(t.columns.review) : '', + strip: fill(DMT.strip, t.columns), + })), + })) + .filter(g => g.tables.length); +} + +/** One row's rendered value, compared with the server's example. */ +async function checkValue(row: Locator, c: any) { + const cell = row.locator('.c-value'); + if (c.verdict === 'clear') { + await expect(cell.locator('.dash')).toBeVisible(); + return; + } + const mv = cell.locator('.mv'); + if (c.example === null || c.example === undefined || c.format === 'null') { + await expect(mv.locator('.mv-null')).toBeVisible(); + return; + } + const ex = String(c.example); + if (TOKEN_RE.test(ex)) { + await expect(mv.locator('.mv-token')).toHaveText(ex); + return; + } + // Masked characters are drawn as bars, kept characters as text. + expect(ex.includes(GLYPH), `${c.id}: example carries the mask`).toBe(true); + const runs = ex.match(new RegExp(`${GLYPH}+`, 'g'))?.length ?? 0; + if (c.format !== 'span') { + await expect(mv.locator('.mv-bar')).toHaveCount(runs); + const kept = (await mv.locator('.mv-seg').allTextContents()).join(''); + expect(kept, `${c.id}: kept characters`).toBe( + ex.replace(new RegExp(GLYPH, 'g'), '') + ); + } else { + expect(await mv.locator('.mv-bar').count()).toBeGreaterThan(0); + } +} + +/** The ledger lines the receipt shows for a column (see the component). */ +function ledgerLines(ev: any): any[] { + const all = ev.lines ?? []; + const scored = all.filter((l: any) => l.points !== 0); + if (!scored.length) return all; + const sum = scored.reduce((s: number, l: any) => s + l.points, 0); + const total = Math.max(0, Math.min(100, ev.total)); + return Math.max(0, Math.min(100, sum)) === total ? scored : all; +} + +/** Transloco-style {{param}} substitution. */ +const fill = (tpl: string, p: Record) => + tpl.replace(/\{\{\s*(\w+)\s*\}\}/g, (_, k) => String(p[k])); + +const pts = (p: number) => (p > 0 ? `+${p}` : p < 0 ? `\u2212${-p}` : '0'); + +test.describe.configure({ mode: 'serial' }); + +test.beforeAll(async ({ browser }: { browser: Browser }, testInfo) => { + page = await browser.newPage({ + viewport: { width: 1440, height: 900 }, + acceptDownloads: true, + baseURL: testInfo.project.use.baseURL, + ignoreHTTPSErrors: true, + }); + page.on('pageerror', e => pageErrors.push(e.message)); + page.on('response', r => { + if ( + r.status() >= 400 && + /\/system\/(data_masking|security_baseline)/.test(r.url()) + ) { + featureErrors.push(`${r.status()} ${r.request().method()} ${r.url()}`); + } + }); + api = await Api.login(page.request); + const [o, access, list] = await Promise.all([ + api.get('/system/data_masking'), + api.get('/system/data_masking/access'), + api.get('/system/security_baseline'), + ]); + snap.mode = o.mode; + snap.admins = { + mode: access.admins.mode, + grants: access.admins.grants.map((g: any) => ({ admin_id: g.admin_id })), + }; + snap.ai = access.ai; + snap.adopted = list.resource + .filter((b: any) => b.adopted) + .map((b: any) => b.id); +}); + +test.afterAll(async () => { + if (!api) { + await page?.close(); + return; + } + const tries: Promise[] = []; + const attempt = (fn: () => Promise) => + tries.push(fn().catch(e => console.warn('restore failed:', String(e)))); + if (snap.admins) { + attempt(() => + api.send('patch', '/system/data_masking/access', { + admins: snap.admins, + ai: snap.ai, + }) + ); + } + attempt(async () => { + const o = await api.get('/system/data_masking'); + // The first scan moves Off to Preview on purpose; keep that. + const want = + snap.mode === 'off' && o.mode === 'preview' ? 'preview' : snap.mode; + if (want && o.mode !== want) { + await api.send('patch', '/system/data_masking', { mode: want }); + } + }); + attempt(async () => { + const list = await api.get('/system/security_baseline'); + for (const b of list.resource) { + if (b.adopted && !snap.adopted?.includes(b.id)) { + await api.send('delete', `/system/security_baseline/${b.id}`); + } + } + }); + await Promise.all(tries); + await page?.close(); +}); + +test('login opens Data Masking (GOLD)', async () => { + await login(page, EMAIL, PASSWORD); + await openDataMasking(); + await expect(page.locator('df-paywall')).toHaveCount(0); +}); + +test('first run: one action and the framework shortcuts', async () => { + const o = await api.get('/system/data_masking'); + const never = + !o.scan.finished_at && o.scan.columns === 0 && o.scan.status !== 'running'; + test.skip(!never, 'Already scanned: the first-run state cannot be shown.'); + await expect(page.getByTestId('dm-no-scan')).toBeVisible(); + await expect(page.getByTestId('empty-state-action')).toHaveText( + DMT.empty.scanAction + ); + // Masking off and nothing scanned: no mode control yet, no counts. + if (o.mode === 'off') { + await expect(page.getByTestId('dm-mode')).toHaveCount(0); + } else { + await expect( + page.getByTestId('dm-mode-on').locator('button') + ).toBeDisabled(); + } + await expect(page.getByTestId('dm-count-masked')).toHaveCount(0); + const chips = page.getByTestId('dm-first-run-baselines').locator('button'); + await expect(chips).toHaveText( + ['HIPAA', 'PCI DSS', 'GLBA', 'CJIS', 'GDPR', 'SOC 2', 'ISO 27001'].map( + v => new RegExp(v) + ) + ); + await shot('01-first-run'); + // A shortcut lands on its baseline. + await page.getByTestId('dm-baseline-hipaa').click(); + await expect(page).toHaveURL(/compliance\?baseline=hipaa/); + await expect(page.locator('#cp-title')).toHaveText('HIPAA baseline', { + timeout: 30_000, + }); + await openDataMasking(); +}); + +test('first run: Apply from a framework shortcut starts the first scan', async () => { + const o = await api.get('/system/data_masking'); + const never = + !o.scan.finished_at && o.scan.columns === 0 && o.scan.status !== 'running'; + test.skip(!never || !APPLY, 'Already scanned (or DM_AUDIT_APPLY=0).'); + const id = FIRST_APPLY_BASELINE; + await page.getByTestId(`dm-baseline-${id}`).click(); + await expect(page).toHaveURL(new RegExp(`compliance\\?baseline=${id}`)); + const b = await api.get(`/system/security_baseline/${id}`); + await expect(page.locator('#cp-title')).toHaveText(`${b.name} baseline`, { + timeout: 30_000, + }); + // Mapped frameworks and the disclaimer sit under the title. + await expect(page.getByTestId('cp-mapped')).toContainText( + 'Not a certification' + ); + // Never scanned: masking.mode cannot pass, and turning masking on is not + // offered (the server would refuse it before a scan). + await page.getByTestId('cp-apply').click(); + const dialog = page.locator('mat-dialog-container'); + await expect(dialog).toBeVisible(); + await expect(dialog.getByTestId('cp-apply-turn-on')).toHaveCount(0); + const applied = page.waitForResponse( + r => + r.url().includes(`/system/security_baseline/${id}/apply`) && + r.request().method() === 'POST' + ); + await dialog.getByTestId('cp-apply-confirm').click(); + const res = await applied; + expect(res.status()).toBe(200); + expect((await res.json()).scan_started).toBe(true); + // The toast says the scan started, and the page follows it to the end. + await expect(page.locator('mat-snack-bar-container')).toContainText( + `${b.name} baseline applied. Scanning` + ); + await expect(page.getByTestId('cp-tracking')).toBeVisible(); + const after = await waitFor( + () => api.get('/system/data_masking'), + v => v.scan.status !== 'running' && !!v.scan.finished_at, + 'first scan (started by Apply) finished', + SCAN_TIMEOUT + ); + await expect(page.getByTestId('cp-scanning')).toHaveCount(0, { + timeout: 30_000, + }); + // The checks were re-read after the scan: the score is the server's. + const d = await api.get(`/system/security_baseline/${id}`); + const pass = d.controls.filter((c: any) => c.status === 'pass').length; + const fail = d.controls.filter((c: any) => c.status === 'fail').length; + await expect(page.getByTestId('cp-score')).toHaveText( + new RegExp(`${pass}\\s+of ${pass + fail}`) + ); + expect(after.scan.columns).toBeGreaterThan(0); + // The scan activity names the baseline that started it. + const act = (await api.get('/system/data_masking/activity?kind=scan')) + .resource; + expect(act.map((e: any) => e.text)).toContain( + `Scan started by the ${b.name} baseline` + ); + await openDataMasking(); +}); + +test('first scan from the page, then Preview without a reload', async () => { + const o = await api.get('/system/data_masking'); + if (!o.scan.finished_at && o.scan.status !== 'running') { + await page.getByTestId('empty-state-action').click(); + await expect(page.getByTestId('dm-first-scan')).toBeVisible(); + } + await expect(page.getByTestId('dm-search')).toBeVisible({ + timeout: SCAN_TIMEOUT, + }); + const after = await waitFor( + () => api.get('/system/data_masking'), + v => v.scan.status !== 'running' && !!v.scan.finished_at, + 'scan finished', + SCAN_TIMEOUT + ); + await expect(page.getByTestId(`dm-mode-${after.mode}`)).toHaveClass( + /mat-button-toggle-checked/ + ); +}); + +test('columns: counts, rail, rows and examples match the server', async () => { + await openDataMasking(); + const m = await model(); + const c = m.overview.counts; + // Disjoint counts, the same as the API, CLI and evidence export. + await expect(page.getByTestId('dm-count-masked')).toContainText( + String(c.masked) + ); + if (c.review) { + await expect(page.getByTestId('dm-count-review')).toContainText( + String(c.review) + ); + } + // To review shows only while there is something to review. + if (c.review) { + await expect(page.getByTestId('dm-filter-review')).toContainText( + String(c.review) + ); + } else { + await expect(page.getByTestId('dm-filter-review')).toHaveCount(0); + } + await expect(page.getByTestId('dm-filter-masked')).toContainText( + String(c.masked) + ); + await expect(page.getByTestId('dm-filter-all')).toContainText( + String(c.masked + c.review + c.clear) + ); + await expect(page.getByTestId('dm-filter-clear')).toContainText( + String(c.clear) + ); + // Server counts add up to the stored columns. + const all = [...m.columns.values()].flat(); + expect(all.length).toBe(c.masked + c.review + c.clear); + expect(all.filter(x => x.verdict === 'review').length).toBe(c.review); + + // Default filter: Review when there is anything to review. + const filter = c.review > 0 ? 'review' : 'masked'; + await expect(page.getByTestId(`dm-filter-${filter}`)).toHaveAttribute( + 'aria-pressed', + 'true' + ); + const keep = + filter === 'review' + ? (x: any) => x.verdict === 'review' + : (x: any) => x.verdict !== 'clear'; + expect(await railOnScreen()).toEqual(expectedRail(m, keep)); + // Scanned databases with nothing under this filter stay listed quietly. + expect(await quietOnScreen()).toEqual(expectedQuiet(m, filter)); + + // The selected (first) table: the queue, then its other columns. + const first = expectedRail(m, keep)[0]; + const svc = m.services.find(s => s.label === first.label); + const cols = tableCols(m, svc.service, first.tables[0].name); + const queue = cols.filter(keep); + const rest = filter === 'review' ? cols.filter(x => !keep(x)) : []; + const expected = [...queue, ...rest]; + await expect(page.locator('.list-title')).toHaveText(first.tables[0].name); + const rows = page.locator('.list [data-testid="dm-row"]'); + await expect(rows).toHaveCount(expected.length); + for (let i = 0; i < expected.length; i++) { + const row = rows.nth(i); + const col = expected[i]; + await expect(row.locator('.c-name code')).toHaveText(col.column); + const label = + col.verdict === 'clear' || !col.class + ? DMT.notSensitive + : DMT.classes[col.class]; + await expect(row.getByTestId('dm-class-chip')).toContainText(label); + await checkValue(row, col); + if (col.verdict === 'review') { + await expect(row.getByTestId('dm-clear')).toBeVisible(); + await expect(row.getByTestId('dm-keep')).toHaveCount(col.class ? 1 : 0); + } + } + await shot('02-columns-review'); + + // Every stored example the API returns renders on its row (All filter). + await page.getByTestId('dm-filter-all').click(); + expect(await railOnScreen()).toEqual(expectedRail(m, () => true)); + let checked = 0; + for (const s of m.services) { + for (const t of s.tables) { + const tc = tableCols(m, s.service, t.name); + if (!tc.some(x => x.verdict !== 'clear')) continue; + await page + .locator('.rail-group') + .filter({ has: page.locator('.rail-service', { hasText: s.label }) }) + .getByTestId('dm-rail-row') + .filter({ + has: page.locator('.rail-name', { + hasText: new RegExp( + `^${t.name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$` + ), + }), + }) + .click(); + await expect(page.locator('.list-title')).toHaveText(t.name); + const r = page.locator('.list [data-testid="dm-row"]'); + await expect(r).toHaveCount(tc.length); + for (let i = 0; i < tc.length; i++) { + if (tc[i].verdict === 'clear') continue; + await checkValue(r.nth(i), tc[i]); + checked++; + } + } + } + expect(checked).toBe(c.masked + c.review); +}); + +test('evidence: the receipt is the server evidence', async () => { + const m = await model(); + const all = [...m.columns.values()].flat(); + // A sampled masked column (values + shapes), a review one, and one whose + // receipt only adds up with its zero-point lines. + const sampled = (c: any, valid: boolean) => + c.verdict === 'masked' && + c.evidence.shapes?.length && + c.evidence.lines.some( + (l: any) => + l.kind === 'values' && + l.points > 0 && + (!valid || /\bvalid\b/.test(l.text)) + ); + const picks = [ + // Prefer one a validator confirmed (checksums, issuer ranges). + all.find(c => sampled(c, true)) ?? all.find(c => sampled(c, false)), + all.find(c => c.verdict === 'review'), + all.find(c => { + const scored = c.evidence.lines.filter((l: any) => l.points !== 0); + const sum = scored.reduce((s: number, l: any) => s + l.points, 0); + return scored.length && Math.min(100, sum) !== c.evidence.total; + }), + ].filter(Boolean); + expect(picks.length).toBeGreaterThan(1); + await page.getByTestId('dm-filter-all').click(); + for (const col of picks) { + await page.getByTestId('dm-search').fill(col.column); + const svc = m.services.find(s => s.service === col.service); + await page + .locator('.rail-group') + .filter({ has: page.locator('.rail-service', { hasText: svc.label }) }) + .getByTestId('dm-rail-row') + .filter({ hasText: col.table }) + .first() + .click(); + await expect(page.locator('.list-title')).toHaveText(col.table); + const row = page + .locator('.list [data-testid="dm-row"]') + .filter({ has: page.locator('.c-name code', { hasText: col.column }) }) + .first(); + await row.getByTestId('dm-expand').click(); + const ledger = page.locator('df-evidence-ledger').first(); + await expect(ledger).toBeVisible(); + const want = ledgerLines(col.evidence); + const lines = ledger.locator('.line:not(.total)'); + await expect(lines.locator('.why')).toHaveText(want.map(l => l.text)); + await expect(lines.locator('.pts')).toHaveText( + want.map(l => pts(l.points)) + ); + await expect(ledger.locator('.line.total .pts')).toHaveText( + String(Math.max(0, Math.min(100, col.evidence.total))) + ); + if (col.evidence.shapes?.length) { + await expect(ledger.locator('.shape code')).toHaveText( + col.evidence.shapes.map((s: any) => s[0]) + ); + } + if (col.type) { + await expect(ledger.locator('.facts dd code').first()).toHaveText( + col.type + ); + } + if (col === picks[0]) await shot('03-evidence', { top: true }); + await row.getByTestId('dm-expand').click(); + } + await page.getByTestId('dm-search').fill(''); +}); + +test('columns in the dark theme', async () => { + await page.evaluate(() => localStorage.setItem('isDarkMode', 'true')); + await openDataMasking(); + const row = page.locator('.list [data-testid="dm-row"]').first(); + await expect(row).toBeVisible(); + await row.getByTestId('dm-expand').click(); + await shot('04-columns-dark'); + await page.evaluate(() => localStorage.setItem('isDarkMode', 'false')); + await openDataMasking(); +}); + +test('mode: turning on asks first, with the real numbers', async () => { + const m = await model(); + const start = m.overview.mode; + test.skip(start === 'on', 'Masking is already on.'); + const total = m.overview.counts.masked + m.overview.counts.review; + const dbs = m.services.filter(s => + s.tables.some((t: any) => t.columns.masked + t.columns.review > 0) + ).length; + await page.getByTestId('dm-mode-on').click(); + const dialog = page.locator('mat-dialog-container'); + await expect(dialog).toContainText(DMT.mode.confirmOn.title); + await expect(dialog).toContainText( + `${total} column${total === 1 ? '' : 's'} in ${dbs} database${dbs === 1 ? '' : 's'}` + ); + await shot('05-mode-on-confirm'); + await dialog.getByRole('button', { name: DMT.cancel }).click(); + await expect(dialog).toHaveCount(0); + await expect(page.getByTestId(`dm-mode-${start}`)).toHaveClass( + /mat-button-toggle-checked/ + ); + expect((await api.get('/system/data_masking')).mode).toBe(start); +}); + +test('access mirrors the server; a grant shows its expiry', async () => { + await dismissToasts(); + await page.getByTestId('dm-tab-access').click(); + await expect(page.getByTestId('dm-access')).toBeVisible(); + const a = await api.get('/system/data_masking/access'); + const modeLabel: Record = { + none: DMT.access.none, + selected: DMT.access.selected, + all: DMT.access.allAdmins, + }; + await expect( + page.getByTestId('dm-admin-mode').locator('.mat-button-toggle-checked') + ).toHaveText(modeLabel[a.admins.mode]); + await expect( + page.getByTestId('dm-ai-mode').locator('.mat-button-toggle-checked') + ).toHaveText(a.ai === 'masked' ? DMT.access.aiMasked : DMT.access.aiSame); + for (const r of a.roles) { + const toggle = page.getByTestId(`dm-role-${r.role_id}`); + await expect(toggle).toContainText(r.name); + await expect(toggle.locator('button[role="switch"]')).toHaveAttribute( + 'aria-checked', + String(r.enabled) + ); + } + const other = a.admins.available.find((x: any) => !x.you); + if (!other || !(await api.get('/system/data_masking')).you.can_grant) { + await shot('06-access'); + return; + } + // Selected: every available admin is listed; grant one for an hour. + if (a.admins.mode !== 'selected') { + await page + .getByTestId('dm-admin-mode') + .getByText(DMT.access.selected) + .click(); + } + const names = page.locator('.admins .admin-name'); + await expect(names).toHaveText(a.admins.available.map((x: any) => x.name)); + await page.getByTestId(`dm-admin-${other.admin_id}`).locator('input').check(); + await waitFor( + () => api.get('/system/data_masking/access'), + x => x.admins.grants.some((g: any) => g.admin_id === other.admin_id), + 'grant saved' + ); + await page.getByLabel(`Access for ${other.name} lasts`).click(); + await page.getByRole('option', { name: DMT.access.durations['1h'] }).click(); + await waitFor( + () => api.get('/system/data_masking/access'), + x => { + const g = x.admins.grants.find((y: any) => y.admin_id === other.admin_id); + return ( + !!g?.expires_at && Date.parse(g.expires_at) - Date.now() < 3_700_000 + ); + }, + 'grant re-timed to 1 hour' + ); + await expect(page.locator('.admin.is-granted .until')).toBeVisible(); + await dismissToasts(); + await shot('06-access'); + // Put it back. + await page + .getByTestId(`dm-admin-${other.admin_id}`) + .locator('input') + .uncheck(); + await waitFor( + () => api.get('/system/data_masking/access'), + x => !x.admins.grants.some((g: any) => g.admin_id === other.admin_id), + 'grant revoked' + ); + if (a.admins.mode !== 'selected') { + await page + .getByTestId('dm-admin-mode') + .getByText(modeLabel[a.admins.mode], { exact: true }) + .click(); + await waitFor( + () => api.get('/system/data_masking/access'), + x => x.admins.mode === a.admins.mode, + 'admins mode restored' + ); + } +}); + +test('access: another admin can take access away, never give it', async ({ + browser, +}) => { + const a = await api.get('/system/data_masking/access'); + const me = a.admins.available.find((x: any) => x.you); + const other = a.admins.available.find((x: any) => x.email === OTHER_EMAIL); + const role = a.roles[0]; + test.skip( + !me || !other || !role, + 'Needs a second admin (DM_OTHER_EMAIL, DM_OTHER_PASSWORD) and a role.' + ); + // As the super admin: something for the other admin to tighten. + await api.send('patch', '/system/data_masking/access', { + admins: { + mode: 'selected', + grants: [ + { admin_id: me.admin_id, duration: '1h' }, + { admin_id: other.admin_id, duration: '8h' }, + ], + }, + roles: [{ role_id: role.role_id, enabled: true, scope: 'all' }], + }); + const ctx = await browser.newContext({ + viewport: { width: 1440, height: 900 }, + baseURL: test.info().project.use.baseURL, + }); + const p2 = await ctx.newPage(); + const denied: string[] = []; + p2.on('response', r => { + if (r.status() >= 400 && /\/system\/data_masking/.test(r.url())) { + denied.push(`${r.status()} ${r.request().method()} ${r.url()}`); + } + }); + const access = () => api.get('/system/data_masking/access'); + try { + await login(p2, OTHER_EMAIL, OTHER_PASSWORD); + await p2.goto(`${DM}?tab=access`); + await expect(p2.getByTestId('dm-access-lock')).toBeVisible({ + timeout: 60_000, + }); + const seg = (group: string, label: string) => + p2 + .getByTestId(group) + .locator('mat-button-toggle') + .filter({ hasText: label }); + await expect(seg('dm-admin-mode', DMT.access.allAdmins)).toHaveClass( + /mat-button-toggle-disabled/ + ); + await expect(seg('dm-ai-mode', DMT.access.aiSame)).toHaveClass( + /mat-button-toggle-disabled/ + ); + // Taking the super admin's own grant away is tightening. + await p2.getByTestId(`dm-admin-${me.admin_id}`).locator('input').uncheck(); + await waitFor( + access, + x => !x.admins.grants.some((g: any) => g.admin_id === me.admin_id), + 'grant removed by the other admin' + ); + // Narrowing a role's scope, then masking it again. + await p2 + .getByRole('combobox', { name: `Real values ${role.name} sees` }) + .click(); + await expect( + p2.getByRole('option', { name: DMT.access.scopeAll }) + ).not.toHaveAttribute('aria-disabled', 'true'); + await p2 + .getByRole('option', { name: DMT.access.scopeExceptRestricted }) + .click(); + await waitFor( + access, + x => + x.roles.find((r: any) => r.role_id === role.role_id)?.scope === + 'except_restricted', + 'role narrowed' + ); + // The panel is gone before the select is opened again. + await expect(p2.getByRole('listbox')).toHaveCount(0); + await p2 + .getByRole('combobox', { name: `Real values ${role.name} sees` }) + .click(); + await expect( + p2.getByRole('option', { name: DMT.access.scopeAll }) + ).toHaveAttribute('aria-disabled', 'true'); + await p2.keyboard.press('Escape'); + await p2 + .getByTestId(`dm-role-${role.role_id}`) + .locator('button[role="switch"]') + .click(); + await waitFor( + access, + x => !x.roles.find((r: any) => r.role_id === role.role_id)?.enabled, + 'role masked again' + ); + await expect( + p2.getByTestId(`dm-role-${role.role_id}`).locator('button[role="switch"]') + ).toBeDisabled(); + // And nobody at all. + await seg('dm-admin-mode', DMT.access.none).click(); + await waitFor( + access, + x => x.admins.mode === 'none', + 'admins set to no one' + ); + await expect(seg('dm-admin-mode', DMT.access.selected)).toHaveClass( + /mat-button-toggle-disabled/ + ); + expect(denied, denied.join('\n')).toEqual([]); + } finally { + await ctx.close(); + await api.send('patch', '/system/data_masking/access', { + admins: snap.admins, + ai: snap.ai, + roles: [ + { role_id: role.role_id, enabled: role.enabled, scope: role.scope }, + ], + }); + } +}); + +test('masks mirror the server', async () => { + await dismissToasts(); + await page.getByTestId('dm-tab-masks').click(); + await expect(page.getByTestId('dm-masks')).toBeVisible(); + const masks = (await api.get('/system/data_masking/mask')).resource; + const tiers = [...new Set(masks.map((x: any) => x.tier))]; + await expect(page.locator('.masks .tier-head')).toHaveText( + tiers.map(t => new RegExp(DMT.tiers[t as string])) + ); + for (const mk of masks) { + await expect(page.getByTestId(`dm-format-${mk.class}`)).toContainText( + DMT.formats[mk.format] + ); + } + // The options are exactly the formats the server allows. + const email = masks.find((x: any) => x.class === 'email'); + await page.getByTestId('dm-format-email').click(); + await expect(page.getByRole('option')).toHaveText( + email.formats_allowed.map((f: string) => DMT.formats[f]) + ); + await page.keyboard.press('Escape'); + await shot('07-masks'); +}); + +test('detection: built-in pack, rules and a live rule test', async () => { + await dismissToasts(); + await page.getByTestId('dm-tab-detection').click(); + await expect(page.getByTestId('dm-detection')).toBeVisible(); + const [builtin, rules] = await Promise.all([ + api.get('/system/data_masking/rule/builtin'), + api.get('/system/data_masking/rule'), + ]); + await expect(page.getByTestId('dm-builtin')).toContainText( + `${builtin.ruleset} ${builtin.version}` + ); + await expect( + page.locator('.rules .row:not(.head-row) code.pattern') + ).toHaveText(rules.resource.map((r: any) => r.pattern)); + await page.getByTestId('dm-builtin').click(); + for (const c of builtin.classes.slice(0, 6)) { + const row = page + .locator('.bi-row') + .filter({ + has: page.locator('.bi-cls', { hasText: DMT.classes[c.class] }), + }) + .first(); + await expect(row.locator('.det')).toHaveText(c.detectors); + } + await page.getByTestId('dm-builtin').click(); + + if (rules.resource.length) await shot('08b-detection-rules', { top: true }); + await page.getByTestId('dm-add-rule').click(); + const pattern = '(^|_)(dob|birth_?date|date_of_birth)$'; + const test = await api.send('post', '/system/data_masking/rule/test', { + match: 'name', + pattern, + }); + await page.getByTestId('dm-rule-pattern').fill(pattern); + const n = test.matches.length; + await expect(page.getByTestId('dm-rule-preview')).toContainText( + `Matches ${n} column${n === 1 ? '' : 's'}`, + { timeout: 15_000 } + ); + await page.getByTestId('dm-rule-class').click(); + await page + .getByRole('option', { name: DMT.classes.date_of_birth, exact: true }) + .click(); + await shot('08-detection-rule'); + await page + .locator('mat-dialog-container') + .getByRole('button', { name: DMT.cancel }) + .click(); + await expect(page.locator('mat-dialog-container')).toHaveCount(0); + expect((await api.get('/system/data_masking/rule')).resource).toEqual( + rules.resource + ); +}); + +test('activity mirrors the server, newest first', async () => { + await dismissToasts(); + // In Preview, an app reading a covered table leaves a "would mask" row. + const o = await api.get('/system/data_masking'); + if (o.mode === 'preview' && PREVIEW_KEY) { + const res = await page.request.get(PREVIEW_PATH, { + headers: { 'X-DreamFactory-API-Key': PREVIEW_KEY }, + }); + expect(res.status(), `GET ${PREVIEW_PATH} with the app key`).toBe(200); + expect(res.headers()['x-dreamfactory-masking']).toBe('preview'); + await res.dispose(); + } + await page.getByTestId('dm-tab-activity').click(); + await expect(page.getByTestId('dm-activity')).toBeVisible(); + const act = (await api.get('/system/data_masking/activity')).resource; + if (act.some((e: any) => e.kind === 'preview')) { + await expect( + page.getByTestId('dm-activity').getByRole('button', { + name: DMT.activity.filters.preview, + exact: true, + }) + ).toBeVisible(); + } + expect(act.length).toBeGreaterThan(0); + const shown = act.slice(0, 30); + const events = page.locator('.event'); + await expect(events.locator('.text')) + .toHaveText( + shown.map( + (e: any) => + new RegExp( + `^\\s*${(e.text + (e.target ? ` ${e.target}` : '')).replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/ /g, '\\s+')}\\s*$` + ) + ), + { useInnerText: true } + ) + .catch(async () => { + // Long lists render in full; compare the leading rows only. + const texts = (await events.locator('.text').allInnerTexts()).slice( + 0, + shown.length + ); + expect(texts.map(t => t.replace(/\s+/g, ' ').trim())).toEqual( + shown.map((e: any) => `${e.text}${e.target ? ` ${e.target}` : ''}`) + ); + }); + const actors = (await events.locator('.actor').allInnerTexts()).slice( + 0, + shown.length + ); + expect(actors).toEqual(shown.map((e: any) => e.actor)); + // Kind filters narrow the list to their kinds. + await page + .getByTestId('dm-activity') + .getByRole('button', { name: DMT.activity.filters.scans, exact: true }) + .click(); + // Each kind is read from the server (kind=…), so wait for its page. + const scans = act.filter((e: any) => e.kind === 'scan').slice(0, 30); + await expect + .poll(async () => + (await events.locator('.text').allInnerTexts()) + .slice(0, scans.length) + .map(t => t.replace(/\s+/g, ' ').trim()) + ) + .toEqual( + scans.map((e: any) => `${e.text}${e.target ? ` ${e.target}` : ''}`) + ); + await expect(page.locator('.event:not(.is-scan)')).toHaveCount(0); + await page + .getByTestId('dm-activity') + .getByRole('button', { name: DMT.activity.filters.all, exact: true }) + .click(); + await shot('09-activity'); +}); + +test('compliance: rail scores and grouping match the server', async () => { + await page.goto(CP); + await expect(page.getByTestId('cp-detail')).toBeVisible({ timeout: 60_000 }); + const list = (await api.get('/system/security_baseline')).resource; + expect(list.map((b: any) => b.id)).toEqual([ + 'hipaa', + 'pci_dss', + 'glba', + 'cjis', + 'gdpr', + 'soc2', + 'iso27001', + ]); + for (const b of list) { + const row = page.getByTestId(`cp-rail-${b.id}`); + await expect(row.locator('.rail-name')).toHaveText(b.name); + await expect(row.locator('.rail-score')).toHaveText( + `${b.score.pass}/${b.score.pass + b.score.fail}` + ); + await expect(row.locator('.rail-tracking')).toHaveCount(b.adopted ? 1 : 0); + } + await expect(page.locator('.rail-vertical')).toHaveText( + [...new Set(list.map((b: any) => b.vertical))].map( + v => new RegExp(CPT.verticals[v as string]) + ) + ); + await expect(page.locator('#cp-title')).toHaveText( + `${(list.find((b: any) => b.adopted) ?? list[0]).name} baseline` + ); + await shot('10-compliance-list'); +}); + +test('compliance: every Fix link opens a real admin page', async () => { + const list = (await api.get('/system/security_baseline')).resource; + const routes = new Set(); + for (const b of list) { + const d = await api.get(`/system/security_baseline/${b.id}`); + for (const c of d.controls) { + if (c.fix?.kind === 'link' && c.fix.route) routes.add(c.fix.route); + } + } + expect(routes.size).toBeGreaterThan(0); + for (const route of routes) { + const [path] = route.split('?'); + await page.goto(`/dreamfactory/dist/#${route}`); + // Unknown routes bounce to home; a real one stays and paints a page. + await page.waitForTimeout(1500); + expect(page.url(), route).toContain(path); + await expect(page.locator('df-paywall')).toHaveCount(0); + await expect( + page.locator('mat-sidenav-content, main').first() + ).toBeVisible(); + } + await page.goto(CP); + await expect(page.getByTestId('cp-detail')).toBeVisible({ timeout: 60_000 }); +}); + +test('compliance: every HIPAA check is the server check', async () => { + await page.getByTestId('cp-rail-gdpr').click(); + await expect(page).toHaveURL(/baseline=gdpr/); + await page.getByTestId('cp-rail-hipaa').click(); + await expect(page).toHaveURL(/baseline=hipaa/); + const d = await api.get('/system/security_baseline/hipaa'); + await expect(page.locator('#cp-title')).toHaveText(`${d.name} baseline`); + const pass = d.controls.filter((c: any) => c.status === 'pass').length; + const fail = d.controls.filter((c: any) => c.status === 'fail').length; + await expect(page.getByTestId('cp-score')).toHaveText( + new RegExp(`${pass}\\s+of ${pass + fail}`) + ); + const manual = d.controls.filter((c: any) => c.status === 'manual'); + if (manual.length) { + await expect(page.getByTestId('cp-manual-toggle')).toContainText( + String(manual.length) + ); + await page.getByTestId('cp-manual-toggle').click(); + } + for (const c of d.controls) { + const row = page.getByTestId(`cp-control-${c.id}`); + await expect(row).toHaveClass(new RegExp(`is-${c.status}`)); + await expect(row.locator('.c-title')).toHaveText(c.title); + await expect(row.locator('.ref')).toHaveText(c.refs); + await expect(row.locator('.c-detail')).toHaveCount( + c.detail && c.status !== 'pass' ? 1 : 0 + ); + if (c.detail && c.status !== 'pass') { + await expect(row.locator('.c-detail')).toHaveText(c.detail); + } + // Every failing check DreamFactory can fix has one action: a link to + // the page that fixes it, or Apply (a button). + const linked = c.status === 'fail' && c.fix?.kind === 'link' && c.fix.route; + const applies = c.status === 'fail' && c.fix?.kind === 'apply'; + await expect(row.getByTestId('cp-fix')).toHaveCount( + linked || applies ? 1 : 0 + ); + if (linked) { + const [path] = c.fix.route.split('?'); + const href = await row.getByTestId('cp-fix').getAttribute('href'); + expect(href ?? '').toContain(path); + } + } + // "Masking is on" failing after a scan: its Fix opens Apply with turning + // masking on already ticked, listing what Apply with masking on changes. + const modeCheck = d.controls.find( + (c: any) => c.id === 'masking.mode' && c.status === 'fail' + ); + const ov = await api.get('/system/data_masking'); + if (modeCheck?.fix?.kind === 'apply' && ov.scan.finished_at) { + const fix = page + .getByTestId('cp-control-masking.mode') + .getByTestId('cp-fix'); + expect(await fix.evaluate(e => e.tagName)).toBe('BUTTON'); + await fix.click(); + const dialog = page.locator('mat-dialog-container'); + await expect( + dialog.getByTestId('cp-apply-turn-on').locator('input') + ).toBeChecked(); + const on = await api.get(`/system/security_baseline/${d.id}?mode=on`); + await expect + .poll(async () => + ( + await dialog + .getByTestId('cp-apply-changes') + .locator('li') + .allInnerTexts() + ).map(t => t.trim()) + ) + .toEqual(on.changes.map((c: any) => c.text)); + await dialog.getByTestId('cp-apply-cancel').click(); + await expect(dialog).toHaveCount(0); + expect((await api.get('/system/data_masking')).mode).toBe(ov.mode); + } + await shot('11-compliance-hipaa', { height: 1240 }); + // The review queue's Fix opens Data Masking on the Review filter. + const review = d.controls.find( + (c: any) => c.id === 'masking.review' && c.status === 'fail' + ); + if (review) { + await page + .getByTestId('cp-control-masking.review') + .getByTestId('cp-fix') + .click(); + await expect(page.getByTestId('dm-filter-review')).toHaveAttribute( + 'aria-pressed', + 'true', + { timeout: 30_000 } + ); + await expect(page).not.toHaveURL(/filter=/); + await page.goBack(); + await expect(page.locator('#cp-title')).toHaveText(`${d.name} baseline`, { + timeout: 30_000, + }); + } +}); + +test('compliance: Apply lists exactly the server changes', async () => { + await dismissToasts(); + const id = APPLY_BASELINE; + await page.getByTestId(`cp-rail-${id}`).click(); + const [before, beforeOn, o] = await Promise.all([ + api.get(`/system/security_baseline/${id}`), + api.get(`/system/security_baseline/${id}?mode=on`), + api.get('/system/data_masking'), + ]); + await expect(page.locator('#cp-title')).toHaveText(`${before.name} baseline`); + test.skip( + before.adopted && !before.changes.length, + 'Already tracked with nothing to change.' + ); + await page.getByTestId('cp-apply').click(); + const dialog = page.locator('mat-dialog-container'); + await expect(dialog).toBeVisible(); + const listed = async () => + ( + await dialog.getByTestId('cp-apply-changes').locator('li').allInnerTexts() + ).map(s => s.trim()); + if (before.changes.length) { + expect(await listed()).toEqual(before.changes.map((c: any) => c.text)); + } else { + await expect(dialog.getByTestId('cp-apply-none')).toBeVisible(); + } + if (o.mode !== 'on' && o.scan?.finished_at) { + // Turning masking on (offered once a scan has finished) lists what + // Apply with mode on would change. + await dialog.getByTestId('cp-apply-turn-on').locator('input').check(); + await expect.poll(listed).toEqual(beforeOn.changes.map((c: any) => c.text)); + await shot('12-compliance-apply-confirm'); + await dialog.getByTestId('cp-apply-turn-on').locator('input').uncheck(); + await expect + .poll(async () => + before.changes.length + ? listed() + : (await dialog.getByTestId('cp-apply-none').count()) === 1 + ) + .toEqual( + before.changes.length ? before.changes.map((c: any) => c.text) : true + ); + } else { + await shot('12-compliance-apply-confirm'); + } + if (!APPLY) { + await dialog.getByTestId('cp-apply-cancel').click(); + return; + } + await dialog.getByTestId('cp-apply-confirm').click(); + await expect(page.getByTestId('cp-tracking')).toBeVisible(); + const after = await waitFor( + () => api.get(`/system/security_baseline/${id}`), + b => b.adopted === true, + 'baseline adopted' + ); + // Exactly the listed changes were made: nothing is left to change. + expect(after.changes).toEqual([]); + // Apply stays only while a check Apply fixes still fails (masking.mode + // needs Apply with masking turned on). + const fixable = after.controls.some( + (c: any) => c.status === 'fail' && c.fix?.kind === 'apply' + ); + await expect(page.getByTestId('cp-apply')).toHaveCount(fixable ? 1 : 0); + const rules = (await api.get('/system/data_masking/rule')).resource; + const added = before.changes.filter((c: any) => c.id.startsWith('rule:')); + expect( + rules.filter((r: any) => r.source === `baseline:${id}`).length + ).toBeGreaterThanOrEqual(added.length); + expect((await api.get('/system/data_masking')).mode).toBe(o.mode); + + // Evidence export: byte-for-byte the server's report. + for (const format of ['json', 'csv'] as const) { + await dismissToasts(); + await page.getByTestId('cp-export').click(); + const dl = page.waitForEvent('download'); + await page.getByTestId(`cp-export-${format}`).click(); + const file = await dl; + const text = readFileSync(await file.path(), 'utf8'); + if (format === 'json') { + const report = JSON.parse(text); + expect(report.baseline?.id ?? report.baseline).toBeTruthy(); + expect(text).not.toMatch(/"generatedAt"/); + for (const c of after.controls) expect(text).toContain(c.id); + } else { + expect(text.trim().split(/\r?\n/).length).toBeGreaterThan( + after.controls.length + ); + } + } + // Stop tracking: settings stay. + if (!snap.adopted?.includes(id)) { + await page.getByTestId('cp-stop').click(); + await expect(page.getByTestId('cp-tracking')).toHaveCount(0); + await waitFor( + () => api.get(`/system/security_baseline/${id}`), + b => b.adopted === false, + 'stopped tracking' + ); + } +}); + +test('narrow screens: columns and compliance at 390 px', async () => { + await page.setViewportSize({ width: 390, height: 844 }); + await openDataMasking(); + await expect(page.locator('.table-select')).toBeVisible(); + await expect( + page.locator('.list [data-testid="dm-row"]').first() + ).toBeVisible(); + const overflow = await page.evaluate( + () => document.documentElement.scrollWidth - window.innerWidth + ); + expect(overflow).toBeLessThanOrEqual(0); + await shot('13-mobile-columns-390'); + await page.goto(`${CP}?baseline=hipaa`); + await expect(page.getByTestId('cp-select')).toBeVisible({ timeout: 60_000 }); + await expect(page.locator('#cp-title')).toHaveText('HIPAA baseline'); + const overflow2 = await page.evaluate( + () => document.documentElement.scrollWidth - window.innerWidth + ); + expect(overflow2).toBeLessThanOrEqual(0); + await shot('14-mobile-compliance-390'); + await page.setViewportSize({ width: 1440, height: 900 }); +}); + +test('no feature API errors and no page errors along the way', async () => { + expect(featureErrors, featureErrors.join('\n')).toEqual([]); + expect(pageErrors, pageErrors.join('\n')).toEqual([]); +}); + +test('open source: Data Masking and Compliance show the paywall', async ({ + browser, +}) => { + test.skip(!OSS_URL, 'Set DM_OSS_BASE_URL to an admin UI on open source.'); + const oss = await browser.newPage({ + viewport: { width: 1440, height: 900 }, + baseURL: OSS_URL, + }); + try { + await login(oss, OSS_EMAIL, OSS_PASSWORD); + for (const [url, name] of [ + [DM, 'Data Masking'], + [CP, 'Compliance'], + ]) { + await oss.goto(url); + await expect(oss.locator('df-paywall')).toBeVisible({ timeout: 60_000 }); + await expect(oss.getByTestId('dm-mode')).toHaveCount(0); + await expect(oss.getByTestId('cp-detail')).toHaveCount(0); + await expect(oss.locator('df-paywall')).toContainText(/\S/); + if (SCREENS) { + await oss.addStyleTag({ + content: + '[id^="intercom"], .intercom-lightweight-app, iframe[name^="intercom"] { display: none !important; }', + }); + await oss.mouse.move(2, 2); + await oss.waitForTimeout(600); + await oss.screenshot({ + path: join( + SCREENS, + name === 'Data Masking' + ? '15-oss-paywall.png' + : '15b-oss-paywall-compliance.png' + ), + caret: 'hide', + }); + } + } + } finally { + await oss.close(); + } +}); diff --git a/e2e/_data-masking-live.config.ts b/e2e/_data-masking-live.config.ts new file mode 100644 index 000000000..19689cae7 --- /dev/null +++ b/e2e/_data-masking-live.config.ts @@ -0,0 +1,27 @@ +/** + * Runs only the Data Masking live journey (real df-masking backend, no + * page.route mocks). The main config ignores `_*` specs, so this one opts + * that single file back in. + * + * # ng serve proxied to a GOLD instance with df-masking installed + * PLAYWRIGHT_BASE_URL=http://127.0.0.1:4300 \ + * DF_ADMIN_EMAIL= DF_ADMIN_PASSWORD= \ + * npx playwright test -c e2e/_data-masking-live.config.ts + * + * Optional: DM_LIVE_BASELINE (default gdpr) picks the baseline to apply; + * DM_LIVE_EVIDENCE_DIR saves screenshots and the downloaded reports. + */ +import { defineConfig } from '@playwright/test'; +import base from '../playwright.config'; + +export default defineConfig({ + ...base, + testDir: '.', + testIgnore: [], + testMatch: ['_data-masking-live.spec.ts'], + // The first scan of every covered database can take minutes. + timeout: 30 * 60_000, + expect: { timeout: 20_000 }, + retries: 0, + workers: 1, +}); diff --git a/e2e/_data-masking-live.spec.ts b/e2e/_data-masking-live.spec.ts new file mode 100644 index 000000000..e70f1c70b --- /dev/null +++ b/e2e/_data-masking-live.spec.ts @@ -0,0 +1,742 @@ +/** + * Data Masking + Compliance, end to end against a REAL df-masking backend + * (no page.route mocks). One serial journey on one page: + * + * login -> first-run scan (or note it already ran) -> review decision -> + * mode change with confirm -> admin access grant -> mask format change -> + * rule test / add / delete -> activity -> compliance apply + export. + * + * Every assertion that matters is checked twice: on screen, and against the + * system API (/api/v2/system/data_masking, /system/security_baseline). + * + * Shared-stack discipline: the journey snapshots what it touches and puts + * it back in afterAll (mode, admin access, AI switch, the email mask + * format, its probe rule, baseline tracking). What it cannot undo, by + * design of the product: a review decision is sticky (it only ever keeps a + * column masked), and Apply's tightening (rules/formats a baseline adds) + * stays. The first scan, when it runs here, is the instance's first scan. + * + * PLAYWRIGHT_BASE_URL=http://127.0.0.1:4300 \ + * npx playwright test -c e2e/_data-masking-live.config.ts + */ +import { + APIRequestContext, + Browser, + Page, + expect, + test, +} from '@playwright/test'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { ADMIN_EMAIL, ADMIN_PASSWORD } from './fixtures/admin-login'; + +const EMAIL = ADMIN_EMAIL; +const PASSWORD = ADMIN_PASSWORD; +const BASELINE = process.env['DM_LIVE_BASELINE'] ?? 'gdpr'; +const EVIDENCE = process.env['DM_LIVE_EVIDENCE_DIR'] ?? ''; +const SCAN_TIMEOUT = Number( + process.env['DM_LIVE_SCAN_TIMEOUT_MS'] ?? 20 * 60_000 +); + +const DM = '/dreamfactory/dist/#/api-security/data-masking'; +const CP = '/dreamfactory/dist/#/api-security/compliance'; +/** A name rule that matches no real column. */ +const PROBE = `^dm_e2e_probe_${Date.now().toString(36)}$`; + +if (EVIDENCE) mkdirSync(EVIDENCE, { recursive: true }); + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +/** Minimal system API client (raw wire format: snake_case). */ +class Api { + constructor( + private ctx: APIRequestContext, + private token: string + ) {} + + static async login(ctx: APIRequestContext): Promise { + const res = await ctx.post('/api/v2/system/admin/session', { + data: { email: EMAIL, password: PASSWORD }, + }); + expect(res.ok(), `admin API login: HTTP ${res.status()}`).toBe(true); + return new Api(ctx, (await res.json()).session_token); + } + + private h() { + return { 'X-DreamFactory-Session-Token': this.token }; + } + + async get(path: string): Promise { + const res = await this.ctx.get(`/api/v2${path}`, { headers: this.h() }); + expect(res.ok(), `GET ${path}: HTTP ${res.status()}`).toBe(true); + return res.json(); + } + + async send( + method: 'patch' | 'post' | 'delete', + path: string, + data?: unknown + ): Promise { + const res = await this.ctx[method](`/api/v2${path}`, { + headers: this.h(), + data, + }); + expect(res.ok(), `${method} ${path}: HTTP ${res.status()}`).toBe(true); + return res.json().catch(() => ({})); + } +} + +let page: Page; +let api: Api; +/** Responses >= 400 from the two feature APIs during the journey. */ +const featureErrors: string[] = []; +const pageErrors: string[] = []; + +/** What the journey found, for afterAll to put back. */ +const snap: { + mode?: string; + admins?: { mode: string; grants: { admin_id: number }[] }; + ai?: string; + emailFormat?: string; + baselineAdopted?: boolean; +} = {}; + +async function evidence(name: string) { + if (!EVIDENCE) return; + await page.waitForTimeout(400); + await page.screenshot({ path: join(EVIDENCE, `${name}.png`), caret: 'hide' }); +} + +async function overview() { + return api.get('/system/data_masking'); +} + +async function waitFor( + read: () => Promise, + ok: (v: T) => boolean, + what: string, + timeout = 20_000 +): Promise { + let last: T | undefined; + await expect + .poll( + async () => { + last = await read(); + return ok(last); + }, + { timeout, message: what } + ) + .toBe(true); + return last as T; +} + +async function openDataMasking(tab = '') { + await page.goto(`${DM}${tab ? `?tab=${tab}` : ''}`); + // Before the first scan with masking off the mode control is not drawn + // (the page leads with the scan), so wait for whichever view loads. + await expect( + page + .getByTestId('dm-mode') + .or(page.getByTestId('dm-no-scan')) + .or(page.getByTestId('dm-first-scan')) + .first() + ).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId('dm-tab-columns')).toHaveText(/\S/); +} + +/** Opens one table of one database in the Columns tab (current filter). */ +async function openTable(service: string, table: string) { + const o = await overview(); + const label = + o.coverage.covered.find((c: any) => c.service === service)?.label ?? + service; + await page + .locator('.rail-group') + .filter({ has: page.locator('.rail-service', { hasText: label }) }) + .getByTestId('dm-rail-row') + .filter({ + has: page.locator('.rail-name', { + hasText: new RegExp( + `^${table.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$` + ), + }), + }) + .click(); + await expect(page.locator('.list-title')).toHaveText(table); +} + +const colRow = (column: string) => + page + .locator('.list [data-testid="dm-row"]') + .filter({ has: page.locator('.c-name code', { hasText: column }) }) + .first(); + +async function dismissToasts() { + await page + .locator('mat-snack-bar-container') + .evaluateAll(els => + els.forEach(e => ((e as HTMLElement).style.display = 'none')) + ); +} + +test.describe.configure({ mode: 'serial' }); + +test.beforeAll(async ({ browser }: { browser: Browser }, testInfo) => { + page = await browser.newPage({ + viewport: { width: 1440, height: 900 }, + acceptDownloads: true, + baseURL: testInfo.project.use.baseURL, + ignoreHTTPSErrors: true, + }); + page.on('pageerror', e => pageErrors.push(e.message)); + page.on('response', r => { + const u = r.url(); + if ( + r.status() >= 400 && + /\/system\/(data_masking|security_baseline)/.test(u) + ) { + featureErrors.push(`${r.status()} ${r.request().method()} ${u}`); + } + }); + api = await Api.login(page.request); + const [o, access, masks, b] = await Promise.all([ + overview(), + api.get('/system/data_masking/access'), + api.get('/system/data_masking/mask'), + api.get(`/system/security_baseline/${BASELINE}`), + ]); + snap.mode = o.mode; + snap.admins = { + mode: access.admins.mode, + grants: access.admins.grants.map((g: any) => ({ admin_id: g.admin_id })), + }; + snap.ai = access.ai; + snap.emailFormat = masks.resource.find( + (m: any) => m.class === 'email' + )?.format; + snap.baselineAdopted = !!b.adopted; +}); + +test.afterAll(async () => { + // Nothing was snapshotted (or changed) without a session. + if (!api) { + await page?.close(); + return; + } + // Best effort, each on its own: never leave the shared stack changed. + const tries: Promise[] = []; + const attempt = (fn: () => Promise) => + tries.push(fn().catch(e => console.warn('restore failed:', String(e)))); + attempt(async () => { + const rules = await api.get('/system/data_masking/rule'); + for (const r of rules.resource.filter((x: any) => x.pattern === PROBE)) { + await api.send('delete', `/system/data_masking/rule/${r.id}`); + } + }); + if (snap.emailFormat) { + attempt(() => + api.send('patch', '/system/data_masking/mask', { + resource: [{ class: 'email', format: snap.emailFormat }], + }) + ); + } + if (snap.admins) { + attempt(() => + api.send('patch', '/system/data_masking/access', { + admins: snap.admins, + ai: snap.ai, + }) + ); + } + if (snap.mode) { + attempt(() => + api.send('patch', '/system/data_masking', { mode: snap.mode }) + ); + } + if (snap.baselineAdopted === false) { + attempt(async () => { + const b = await api.get(`/system/security_baseline/${BASELINE}`); + if (b.adopted) { + await api.send('delete', `/system/security_baseline/${BASELINE}`); + } + }); + } + await Promise.all(tries); + await page?.close(); +}); + +test('login opens Data Masking (GOLD)', async () => { + await page.context().clearCookies(); + await page.goto('/dreamfactory/dist/#/auth/login'); + const email = page.locator('input').first(); + await expect(email).toBeVisible({ timeout: 30_000 }); + await email.fill(EMAIL); + await page.locator('input').nth(1).fill(PASSWORD); + const login = page.waitForResponse( + r => + r.url().includes('/api/v2/system/admin/session') && + r.request().method() === 'POST' + ); + await page.getByRole('button', { name: /^login$/i }).click(); + expect((await login).ok()).toBe(true); + await page.waitForURL(u => !u.toString().includes('/auth/login'), { + timeout: 30_000, + }); + await openDataMasking(); + await expect(page.locator('df-paywall')).toHaveCount(0); +}); + +test('first run: one scan, then Preview without a reload', async () => { + const o = await overview(); + const neverScanned = + !o.scan.finished_at && o.scan.columns === 0 && o.scan.status !== 'running'; + if (neverScanned) { + await expect(page.getByTestId('dm-no-scan')).toBeVisible(); + await expect(page.getByTestId('dm-first-run-baselines')).toBeVisible(); + // Masking off and nothing scanned: the page leads with the scan; the + // mode control waits (Off stays reachable if masking was left on). + if (o.mode === 'off') { + await expect(page.getByTestId('dm-mode')).toHaveCount(0); + } else { + await expect( + page.getByTestId('dm-mode-off').locator('button') + ).toBeEnabled(); + await expect( + page.getByTestId('dm-mode-on').locator('button') + ).toBeDisabled(); + } + await evidence('01-first-run'); + await page.getByTestId('empty-state-action').click(); + await expect(page.getByTestId('dm-first-scan')).toBeVisible(); + await evidence('02-first-scan-running'); + } else { + test.info().annotations.push({ + type: 'note', + description: `Already scanned (finished_at=${o.scan.finished_at}); the never-scanned state cannot be shown on this instance.`, + }); + } + // Whatever started the scan, the page follows it to the end, live. + await expect(page.getByTestId('dm-search')).toBeVisible({ + timeout: SCAN_TIMEOUT, + }); + const after = await waitFor( + overview, + v => v.scan.status !== 'running' && !!v.scan.finished_at, + 'scan finished', + SCAN_TIMEOUT + ); + expect(after.scan.columns).toBeGreaterThan(0); + if (neverScanned && o.mode === 'off') { + // The server moves Off -> Preview after the first scan ever. + expect(after.mode).toBe('preview'); + await expect(page.getByTestId('dm-mode-preview')).toHaveClass( + /mat-button-toggle-checked/ + ); + } + // Header counts are the server's, disjoint: masked, then to review. + await expect(page.getByTestId('dm-count-masked')).toContainText( + String(after.counts.masked) + ); + snap.mode = after.mode; + await evidence('03-scanned'); +}); + +/** + * Review decisions are made on purpose, not on whatever row comes first: + * free-text clinical notes in a patient table stay masked, and a sign-up + * date the value shapes mistook for a birth date is marked not sensitive + * (its Undo is tried, then the decision is made again). Override with + * DM_LIVE_KEEP / DM_LIVE_CLEAR (service.table.column). + */ +const KEEP_ID = + process.env['DM_LIVE_KEEP'] ?? 'crm.patient_encounters.clinical_notes'; +const CLEAR_ID = process.env['DM_LIVE_CLEAR'] ?? 'crm.clientes.fecha_alta'; + +async function column(id: string): Promise { + const [service] = id.split('.'); + const r = await api.get( + `/system/data_masking/column?service=${encodeURIComponent(service)}` + ); + return r.resource.find((c: any) => c.id === id); +} + +test('review: deliberate decisions (keep notes masked, a sign-up date is not a birth date)', async () => { + const before = await overview(); + const [keep, clear] = await Promise.all([column(KEEP_ID), column(CLEAR_ID)]); + test.skip( + keep?.verdict !== 'review' || clear?.verdict !== 'review', + `${KEEP_ID} and ${CLEAR_ID} are not both waiting for review here.` + ); + // Column decisions go out with the id in the body (any column name works). + const decisions: { url: string; body: any }[] = []; + const onRequest = (r: any) => { + if ( + r.method() === 'PATCH' && + /\/system\/data_masking\/column/.test(r.url()) + ) { + decisions.push({ url: r.url(), body: r.postDataJSON() }); + } + }; + page.on('request', onRequest); + try { + await dismissToasts(); + await page.getByTestId('dm-filter-review').click(); + + // 1. Clinical notes: free text about patients stays masked. + await openTable(keep.service, keep.table); + await colRow(keep.column).getByTestId('dm-keep').click(); + await expect(colRow(keep.column).getByTestId('dm-decided')).toBeVisible(); + const kept = await waitFor( + () => column(KEEP_ID), + c => c.verdict === 'masked' && c.source === 'admin', + `${KEEP_ID} kept masked by the admin` + ); + expect(kept.class).toBe(keep.class); + + // 2. fecha_alta is when the customer signed up: not sensitive. + await openTable(clear.service, clear.table); + await colRow(clear.column).getByTestId('dm-clear').click(); + const toast = page.locator('df-dm-undo-toast'); + await expect(toast).toContainText(`${clear.column} marked not sensitive`); + await waitFor( + () => column(CLEAR_ID), + c => c.verdict === 'clear' && c.source === 'admin', + `${CLEAR_ID} marked not sensitive` + ); + // Undo masks it again with the class it had... + await toast.getByTestId('dm-undo').click(); + await waitFor( + () => column(CLEAR_ID), + c => c.verdict === 'masked' && c.class === clear.class, + `${CLEAR_ID} masked again by Undo` + ); + // ...and the admin then makes the decision that is actually right. + await page.getByTestId('dm-filter-all').click(); + await openTable(clear.service, clear.table); + await colRow(clear.column).getByTestId('dm-class-chip').click(); + await page.getByTestId('dm-class-none').click(); + await waitFor( + () => column(CLEAR_ID), + c => c.verdict === 'clear' && c.source === 'admin', + `${CLEAR_ID} not sensitive again` + ); + await waitFor( + overview, + v => v.counts.review === before.counts.review - 2, + 'review count went down by two' + ); + await dismissToasts(); + await evidence('04-review-decided'); + expect(decisions.length).toBeGreaterThanOrEqual(4); + for (const d of decisions) { + expect(d.body?.id, d.url).toMatch(/^[^.]+\.[^.]+\..+$/); + expect(new URL(d.url).pathname).toMatch( + /\/system\/data_masking\/column$/ + ); + } + } finally { + page.off('request', onRequest); + } +}); + +test('mode: On asks first with the real numbers, then back', async () => { + await dismissToasts(); + const start = (await overview()).mode as string; + // Confirms come whenever masking is added or removed: turning On, and + // leaving On (for Preview or Off). Preview and Off swap without one. + const go = async (mode: string, confirm: boolean) => { + await page.getByTestId(`dm-mode-${mode}`).click(); + if (confirm) { + await expect(page.getByTestId('dm-confirm-ok')).toBeVisible(); + await page.getByTestId('dm-confirm-ok').click(); + } else { + await expect(page.getByTestId('dm-confirm-ok')).toHaveCount(0); + } + await expect(page.getByTestId(`dm-mode-${mode}`)).toHaveClass( + /mat-button-toggle-checked/ + ); + await waitFor(overview, v => v.mode === mode, `mode is ${mode}`); + }; + if (start === 'on') await go('preview', true); + await page.getByTestId('dm-mode-on').click(); + const dialog = page.locator('mat-dialog-container'); + await expect(dialog).toContainText('Turn on masking?'); + await expect(dialog).toContainText(/\d+ columns? in \d+ databases?/); + await evidence('05-mode-on-confirm'); + await page.getByTestId('dm-confirm-ok').click(); + await expect(page.getByTestId('dm-mode-on')).toHaveClass( + /mat-button-toggle-checked/ + ); + await waitFor(overview, v => v.mode === 'on', 'mode is on'); + // Restore what the journey started from. + if (start === 'preview') await go('preview', true); + if (start === 'off') await go('off', true); +}); + +test('access: grant an admin real values, re-time it, revoke it', async () => { + await dismissToasts(); + const access = await api.get('/system/data_masking/access'); + const other = access.admins.available.find((a: any) => !a.you); + test.skip(!other, 'No second admin on this instance'); + const canGrant = (await overview()).you.can_grant; + test.skip(!canGrant, 'Signed-in admin cannot give access (not super admin)'); + await page.getByTestId('dm-tab-access').click(); + await expect(page.getByTestId('dm-access')).toBeVisible(); + await expect(page.getByTestId('dm-access-lock')).toHaveCount(0); + if (access.admins.mode !== 'selected') { + await page.getByTestId('dm-admin-mode').getByText('Selected').click(); + await waitFor( + () => api.get('/system/data_masking/access'), + a => a.admins.mode === 'selected', + 'admins mode selected' + ); + } + const box = page.getByTestId(`dm-admin-${other.admin_id}`).locator('input'); + await box.check(); + const granted = await waitFor( + () => api.get('/system/data_masking/access'), + a => a.admins.grants.some((g: any) => g.admin_id === other.admin_id), + 'grant saved' + ); + const g8 = granted.admins.grants.find( + (g: any) => g.admin_id === other.admin_id + ); + // New grants default to 8 hours. + expect(g8.expires_at).not.toBeNull(); + expect(Date.parse(g8.expires_at) - Date.now()).toBeLessThanOrEqual( + 8 * 3_600_000 + 120_000 + ); + await page.getByLabel(`Access for ${other.name} lasts`).click(); + await page.getByRole('option', { name: '1 hour' }).click(); + await waitFor( + () => api.get('/system/data_masking/access'), + a => { + const g = a.admins.grants.find((x: any) => x.admin_id === other.admin_id); + return ( + !!g?.expires_at && + Date.parse(g.expires_at) - Date.now() <= 3_600_000 + 120_000 + ); + }, + 'grant re-timed to 1 hour' + ); + await dismissToasts(); + await evidence('06-access-granted'); + await box.uncheck(); + await waitFor( + () => api.get('/system/data_masking/access'), + a => !a.admins.grants.some((g: any) => g.admin_id === other.admin_id), + 'grant revoked' + ); + if (snap.admins && snap.admins.mode !== 'selected') { + const label = { none: 'No one', all: 'All admins' }[snap.admins.mode]; + if (label) { + await page.getByTestId('dm-admin-mode').getByText(label).click(); + await waitFor( + () => api.get('/system/data_masking/access'), + a => a.admins.mode === snap.admins!.mode, + 'admins mode restored' + ); + } + } +}); + +test('masks: changing a format re-renders the example and the columns', async () => { + await dismissToasts(); + await page.getByTestId('dm-tab-masks').click(); + await expect(page.getByTestId('dm-masks')).toBeVisible(); + const from = snap.emailFormat ?? 'token'; + const to = from === 'initial' ? 'token' : 'initial'; + const label = { initial: 'First letter', token: 'Token' }[to]; + await page.getByTestId('dm-format-email').click(); + await page.getByRole('option', { name: label }).click(); + await waitFor( + () => api.get('/system/data_masking/mask'), + m => m.resource.find((x: any) => x.class === 'email')?.format === to, + `email masked as ${to}` + ); + // Stored examples follow the class format on the server too. + const cols = await waitFor( + () => api.get('/system/data_masking/column?service=crm'), + c => + c.resource + .filter((x: any) => x.class === 'email' && x.verdict !== 'clear') + .every((x: any) => x.format === to), + 'email columns report the new format' + ); + const emails = cols.resource.filter( + (x: any) => x.class === 'email' && x.verdict !== 'clear' && x.example + ); + for (const e of emails) { + if (to === 'token') + expect(e.example).toMatch(/^[A-Z][A-Z0-9]*_[0-9a-f]{8}$/); + else expect(e.example).toContain('•'); + } + await dismissToasts(); + await evidence('07-masks-email-changed'); + // Back to what it was. + const back = + { initial: 'First letter', token: 'Token', redact: 'Redact', null: 'Null' }[ + from + ] ?? from; + await page.getByTestId('dm-format-email').click(); + await page.getByRole('option', { name: back, exact: true }).click(); + await waitFor( + () => api.get('/system/data_masking/mask'), + m => m.resource.find((x: any) => x.class === 'email')?.format === from, + 'email format restored' + ); +}); + +test('detection: live rule test, add a rule, delete it', async () => { + await dismissToasts(); + await page.getByTestId('dm-tab-detection').click(); + await expect(page.getByTestId('dm-detection')).toBeVisible(); + await page.getByTestId('dm-add-rule').click(); + const pattern = page.getByTestId('dm-rule-pattern'); + // A pattern that matches real columns: the preview counts them live. + await pattern.fill('e_?mail'); + await expect(page.getByTestId('dm-rule-preview')).toContainText( + /Matches (at least )?[1-9]\d* columns?/, + { timeout: 15_000 } + ); + await pattern.fill(PROBE); + await expect(page.getByTestId('dm-rule-preview')).toContainText( + 'Matches 0 columns', + { timeout: 15_000 } + ); + await page.getByTestId('dm-rule-class').click(); + await page.getByRole('option', { name: 'Email', exact: true }).click(); + await page.getByTestId('dm-rule-save').click(); + await expect(page.locator('mat-dialog-container')).toHaveCount(0); + const row = page.locator('.rules .row').filter({ hasText: PROBE }); + await expect(row).toBeVisible(); + await waitFor( + () => api.get('/system/data_masking/rule'), + r => r.resource.some((x: any) => x.pattern === PROBE), + 'rule saved' + ); + await evidence('08-rule-added'); + await row.getByRole('button', { name: 'Delete rule' }).click(); + // Delete asks first and says the rule's columns stay masked. + await expect(page.locator('mat-dialog-container')).toContainText( + 'Delete rule?' + ); + await page.getByTestId('dm-confirm-ok').click(); + await expect(row).toHaveCount(0); + await waitFor( + () => api.get('/system/data_masking/rule'), + r => !r.resource.some((x: any) => x.pattern === PROBE), + 'rule deleted' + ); +}); + +test('activity lists what this journey did, newest first', async () => { + await dismissToasts(); + await page.getByTestId('dm-tab-activity').click(); + await expect(page.getByTestId('dm-activity')).toBeVisible(); + const act = await api.get('/system/data_masking/activity?limit=50'); + const kinds = new Set(act.resource.map((e: any) => e.kind)); + for (const k of ['mode', 'access', 'rule']) expect(kinds).toContain(k); + // The newest server row is on screen. + await expect(page.locator('.event').first()).toContainText( + act.resource[0].text.slice(0, 40) + ); + await page.getByRole('button', { name: 'Access', exact: true }).click(); + await expect(page.locator('.event').first()).toBeVisible(); + await evidence('09-activity'); +}); + +test('compliance: apply lists exactly the server changes; export JSON and CSV', async () => { + await dismissToasts(); + const before = await api.get(`/system/security_baseline/${BASELINE}`); + await page.goto(`${CP}?baseline=${BASELINE}`); + await expect(page.locator('#cp-title')).toHaveText( + `${before.name} baseline`, + { + timeout: 60_000, + } + ); + // The score is the server's pass / (pass + fail). + const pass = before.controls.filter((c: any) => c.status === 'pass').length; + const fail = before.controls.filter((c: any) => c.status === 'fail').length; + await expect(page.getByTestId('cp-score')).toHaveText( + new RegExp(`${pass}\\s+of ${pass + fail}`) + ); + await evidence('10-compliance'); + + if (!before.adopted || before.changes.length) { + await page.getByTestId('cp-apply').click(); + const dialog = page.locator('mat-dialog-container'); + await expect(dialog).toBeVisible(); + if (before.changes.length) { + const shown = await page + .getByTestId('cp-apply-changes') + .locator('li') + .allInnerTexts(); + expect(shown.map(s => s.trim())).toEqual( + before.changes.map((c: any) => c.text) + ); + } else { + await expect(page.getByTestId('cp-apply-none')).toBeVisible(); + } + await evidence('11-compliance-apply-dialog'); + await page.getByTestId('cp-apply-confirm').click(); + await expect(page.getByTestId('cp-tracking')).toBeVisible(); + const after = await api.get(`/system/security_baseline/${BASELINE}`); + expect(after.adopted).toBe(true); + expect(after.changes.length).toBeLessThanOrEqual(before.changes.length); + // Apply stays while anything is left to change or a check Apply fixes + // still fails (masking.mode needs Apply with masking turned on). + const fixable = after.controls.some( + (c: any) => c.status === 'fail' && c.fix?.kind === 'apply' + ); + await expect(page.getByTestId('cp-apply')).toHaveCount( + after.changes.length || fixable ? 1 : 0 + ); + await evidence('12-compliance-applied'); + } + + for (const format of ['json', 'csv'] as const) { + await dismissToasts(); + await page.getByTestId('cp-export').click(); + const dl = page.waitForEvent('download'); + await page.getByTestId(`cp-export-${format}`).click(); + const file = await dl; + expect(file.suggestedFilename()).toMatch( + new RegExp( + `^dreamfactory-${BASELINE}-baseline-\\d{4}-\\d{2}-\\d{2}\\.${format}$` + ) + ); + const path = await file.path(); + const text = readFileSync(path!, 'utf8'); + if (EVIDENCE) writeFileSync(join(EVIDENCE, file.suggestedFilename()), text); + const ids = before.controls.map((c: any) => c.id); + if (format === 'json') { + const report = JSON.parse(text); + // Saved verbatim: the server's snake_case keys, not camelCased. + expect(text).not.toMatch(/"generatedAt"/); + const reported = JSON.stringify(report); + for (const id of ids) expect(reported).toContain(id); + } else { + const lines = text.trim().split(/\r?\n/); + expect(lines.length).toBeGreaterThanOrEqual(ids.length + 1); + for (const id of ids) expect(text).toContain(id); + } + } + + if (snap.baselineAdopted === false) { + await page.getByTestId('cp-stop').click(); + await expect(page.getByTestId('cp-tracking')).toHaveCount(0); + await waitFor( + () => api.get(`/system/security_baseline/${BASELINE}`), + b => b.adopted === false, + 'baseline no longer tracked' + ); + } +}); + +test('no feature API errors and no page errors along the way', async () => { + expect(featureErrors, featureErrors.join('\n')).toEqual([]); + expect(pageErrors, pageErrors.join('\n')).toEqual([]); +}); diff --git a/e2e/_data-masking-r2.spec.ts b/e2e/_data-masking-r2.spec.ts new file mode 100644 index 000000000..87a654239 --- /dev/null +++ b/e2e/_data-masking-r2.spec.ts @@ -0,0 +1,713 @@ +/** + * Data Masking round-2 flows against the contract mock (exploratory: the + * leading underscore keeps it out of CI). Each test drives one fix and saves + * a screenshot of the screen it changed. + * + * DM_SPEC=_data-masking-r2.spec.ts PLAYWRIGHT_BASE_URL=http://127.0.0.1:4321 \ + * DF_ADMIN_EMAIL=... DF_ADMIN_PASSWORD=... DM_SCREENS_DIR=... \ + * npx playwright test -c e2e/_data-masking-screens.config.ts + */ +import { Page, Request, expect, test } from '@playwright/test'; +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { loginAsAdmin } from './fixtures/admin-login'; +import { + DataMaskingMockOptions, + installDataMaskingMock, +} from './fixtures/data-masking-mock'; + +const OUT = process.env['DM_SCREENS_DIR'] ?? 'test-results/data-masking-r2'; +mkdirSync(OUT, { recursive: true }); +const PAGE = '/dreamfactory/dist/#/api-security/data-masking'; +type Theme = 'light' | 'dark'; + +test.describe.configure({ mode: 'default' }); +test.setTimeout(120_000); + +async function boot( + page: Page, + opts: DataMaskingMockOptions = {}, + theme: Theme = 'light' +) { + await page.addInitScript(t => { + localStorage.setItem('isDarkMode', JSON.stringify(t === 'dark')); + localStorage.setItem('isPhosphor', 'false'); + }, theme); + const state = await installDataMaskingMock(page, opts); + await loginAsAdmin(page); + return state; +} + +async function open(page: Page, tab = '') { + await page.goto(`${PAGE}${tab ? `?tab=${tab}` : ''}`); + await expect( + page + .locator( + '[data-testid="dm-mode"], [data-testid="dm-no-scan"], [data-testid="dm-no-databases"]' + ) + .first() + ).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId('dm-tab-columns')).toHaveText(/\S/); +} + +async function shot(page: Page, name: string) { + await page.addStyleTag({ + content: + '[id^="intercom"], .intercom-lightweight-app, iframe[name^="intercom"] { display: none !important; }', + }); + await page.waitForTimeout(400); + await page.screenshot({ path: join(OUT, `${name}.png`), caret: 'hide' }); +} + +const isPatch = (path: RegExp) => (r: Request) => + r.method() === 'PATCH' && path.test(new URL(r.url()).pathname); + +test('mode: leaving On asks first; Preview to Off does not', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { mode: 'on' }); + await open(page); + await page.getByTestId('dm-mode-preview').click(); + await expect(page.getByRole('dialog')).toContainText('Stop masking?'); + await shot(page, 'r2-mode-on-to-preview-confirm'); + const patch = page.waitForRequest(isPatch(/\/system\/data_masking$/)); + await page.getByTestId('dm-confirm-ok').click(); + expect((await patch).postDataJSON()).toEqual({ mode: 'preview' }); + await expect(page.getByText('Masking stopped. Preview is on.')).toBeVisible(); + // Preview to Off: nothing was masked, so no dialog. + const off = page.waitForRequest(isPatch(/\/system\/data_masking$/)); + await page.getByTestId('dm-mode-off').click(); + expect((await off).postDataJSON()).toEqual({ mode: 'off' }); + await expect(page.getByRole('dialog')).toHaveCount(0); +}); + +test('mode: the turn-on confirm says admins are masked and what is left', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + // Kevin has no grant here, so he is masked too. + const state = await boot(page); + state.access.admins.grants = []; + await open(page); + // Header counts are disjoint: masked, then to review. + await expect(page.getByTestId('dm-count-masked')).toContainText('182'); + await expect(page.getByTestId('dm-count-review')).toContainText('24'); + await page.getByTestId('dm-mode-on').click(); + await expect(page.getByRole('dialog')).toContainText( + 'will be masked for apps, AI tools and admins, you included. 24 still to review.' + ); + await shot(page, 'r2-mode-on-confirm'); + await page.getByTestId('dm-confirm-cancel').click(); +}); + +test('mode: an admin who cannot lower it sees Off and Preview locked', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { scenario: 'nogrant', mode: 'on' }); + await open(page); + await expect(page.getByTestId('dm-mode-lock')).toHaveText( + /Only the super admin can lower the masking mode\./ + ); + await expect( + page.getByTestId('dm-mode-off').locator('button') + ).toBeDisabled(); + await expect( + page.getByTestId('dm-mode-preview').locator('button') + ).toBeDisabled(); + await shot(page, 'r2-mode-lowering-locked'); +}); + +test('mode: nothing is shown as selected while the overview loads', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { mode: 'on' }); + // Hold the overview so the loading state can be seen. + let release: () => void = () => undefined; + const held = new Promise(r => (release = r)); + await page.route(/\/api\/v2\/system\/data_masking$/, async route => { + await held; + await route.fallback(); + }); + await page.goto(PAGE); + await expect(page.getByTestId('dm-mode-loading')).toBeVisible({ + timeout: 60_000, + }); + await expect(page.getByTestId('dm-mode')).toHaveCount(0); + await shot(page, 'r2-mode-loading'); + release(); + await expect(page.getByTestId('dm-mode-on')).toHaveClass( + /mat-button-toggle-checked/ + ); +}); + +test('columns: disjoint chips, All with its count; Not sensitive has Undo', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page); + await open(page); + const chip = (f: string) => page.getByTestId(`dm-filter-${f}`); + await expect(chip('review')).toContainText('24'); + await expect(chip('masked')).toContainText('182'); + await expect(chip('clear')).toContainText('447'); + await expect(chip('all')).toContainText('653'); + const patch = page.waitForRequest(isPatch(/\/system\/data_masking\/column$/)); + const row = page.getByTestId('dm-row').first(); + const name = (await row.locator('.c-name code').textContent())?.trim(); + await row.getByTestId('dm-clear').click(); + const body = (await patch).postDataJSON(); + expect(body.class).toBeNull(); + expect(body.id).toMatch(/\./); + // Focus stays in the row, so keyboard review continues. + expect( + await page.evaluate(() => + document.activeElement?.getAttribute('data-testid') + ) + ).toBe('dm-row'); + await expect(page.getByText(`${name} marked not sensitive`)).toBeVisible(); + await shot(page, 'r2-not-sensitive-undo'); + const undo = page.waitForRequest(isPatch(/\/system\/data_masking\/column$/)); + await page.getByTestId('dm-undo').click(); + expect((await undo).postDataJSON().class).not.toBeNull(); +}); + +test('scan: a database that cannot be read is named, with Retry', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { unreachable: true }); + await open(page); + await page.getByTestId('dm-scan').click(); + await expect( + page.getByText("Scan finished. Couldn't read Orders (MySQL).") + ).toBeVisible({ timeout: 30_000 }); + await page.getByTestId('dm-filter-masked').click(); + const failed = page.getByTestId('dm-rail-failed'); + await failed.scrollIntoViewIfNeeded(); + await expect(failed).toContainText("Couldn't scan"); + await expect(failed).toContainText('Orders (MySQL)'); + await shot(page, 'r2-scan-partial'); +}); + +test('first run: a default install with only the empty built-in db', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { scenario: 'emptydb' }); + await open(page); + await expect(page.getByTestId('dm-no-databases')).toContainText( + 'No tables to scan' + ); + await expect(page.getByTestId('dm-no-scan')).toHaveCount(0); + await expect(page.getByTestId('dm-baseline-iso27001')).toHaveText( + /ISO 27001/ + ); + await expect(page.getByTestId('dm-mode')).toHaveCount(0); + await shot(page, 'r2-first-run-empty-db'); +}); + +test('access: All admins and AI asks first; a stale page is refused', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + const state = await boot(page); + await open(page, 'access'); + await expect(page.getByTestId('dm-access')).toBeVisible(); + await page.getByTestId('dm-admin-mode').getByText('All admins').click(); + await expect(page.getByRole('dialog')).toContainText( + 'All admins will see real values in 206 columns.' + ); + await shot(page, 'r2-access-all-admins-confirm'); + await page.getByTestId('dm-confirm-cancel').click(); + await expect( + page.getByTestId('dm-admin-mode').locator('.mat-button-toggle-checked') + ).toHaveText('Selected'); + await page.getByTestId('dm-ai-mode').getByText('Same as admins').click(); + await expect(page.getByRole('dialog')).toContainText( + 'AI and MCP sessions will see real values' + ); + await page.getByTestId('dm-confirm-cancel').click(); + + // Someone else changes access (the CLI revokes every grant). + state.access.admins.grants = []; + state.accessVersion++; + const patch = page.waitForRequest(isPatch(/\/data_masking\/access$/)); + await page.getByTestId('dm-role-3').locator('button').click(); + const body = (await patch).postDataJSON(); + // Only the changed role, with the version the page read. + expect(body.roles).toEqual([ + { role_id: 3, enabled: true, scope: 'except_restricted' }, + ]); + expect(body.version).toBe('v1'); + // The page re-reads by itself, and says so (not the server's "Reload"). + await expect( + page.getByText('Access settings changed elsewhere. Showing the latest.') + ).toBeVisible(); + // Re-read: the revoked grant is not brought back. + await expect( + page.getByTestId('dm-admin-1').locator('input') + ).not.toBeChecked(); + await shot(page, 'r2-access-stale-refused'); +}); + +test('access: durations past a tracked baseline are marked', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page); + // HIPAA tracked: the mock's baseline list says so. + await page.route('**/api/v2/system/security_baseline', route => + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + resource: [{ id: 'hipaa', name: 'HIPAA', adopted: true }], + }), + }) + ); + await open(page, 'access'); + await page.locator('.admin.is-granted mat-select').first().click(); + await expect(page.getByTestId('dm-duration-exceeds').first()).toHaveText( + 'Exceeds HIPAA' + ); + await shot(page, 'r2-access-duration-exceeds'); + await page.keyboard.press('Escape'); +}); + +test('activity: chips ask the server; Load older pages back', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + const state = await boot(page); + for (let i = 0; i < 260; i++) { + state.ev( + i * 1000, + 'mobile-app', + 'preview', + 'Would mask 3 columns', + `sports.t${i}` + ); + } + await open(page, 'activity'); + await expect(page.getByTestId('dm-activity-older')).toBeVisible(); + const req = page.waitForRequest( + r => + r.url().includes('/data_masking/activity') && + r.url().includes('kind=blocked') + ); + await page.getByTestId('dm-activity-filter-blocked').click(); + await req; + await expect(page.locator('.event')).toHaveCount(2); + await shot(page, 'r2-activity-blocked'); + await page.getByTestId('dm-activity-filter-all').click(); + const older = page.waitForRequest( + r => + r.url().includes('/data_masking/activity') && r.url().includes('before=') + ); + await page.getByTestId('dm-activity-older').click(); + await older; + await expect(page.getByTestId('dm-activity-older')).toHaveCount(0); +}); + +test('detection: delete asks and offers to unmask what only it masked', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page); + await open(page, 'detection'); + await page.getByTestId('dm-add-rule').click(); + await page.getByTestId('dm-rule-pattern').fill('(^|_)sku$'); + await page.getByTestId('dm-rule-class').click(); + await page.getByRole('option', { name: 'Username' }).click(); + await page.getByTestId('dm-rule-save').click(); + await expect(page.getByRole('dialog')).toHaveCount(0); + await page.getByTestId('dm-rule-delete').last().click(); + await expect(page.getByRole('dialog')).toContainText('Delete rule?'); + await shot(page, 'r2-rule-delete-confirm'); + await page.getByTestId('dm-confirm-cancel').click(); + // A PCRE-only pattern is the server's to judge, not refused here. + await page.getByTestId('dm-add-rule').click(); + await page.getByTestId('dm-rule-pattern').fill('^(?>emp|staff)_ssn$'); + await page.getByTestId('dm-rule-class').click(); + await page.getByRole('option', { name: 'National ID' }).click(); + await expect(page.getByTestId('dm-rule-error')).toHaveCount(0); + await expect(page.getByTestId('dm-rule-save')).toBeEnabled(); + await page.keyboard.press('Escape'); +}); + +test('background: a database scanned while the page is open shows up', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + const state = await boot(page); + await open(page); + await page.getByTestId('dm-filter-all').click(); + // The server scans a new database on its own (added in API Generation). + state.pending = true; + state.scan.finished_at = new Date().toISOString(); + await page.evaluate(() => window.dispatchEvent(new Event('focus'))); + await expect(page.getByTestId('dm-rail-unscanned')).toContainText( + 'HR (SQL Server)', + { timeout: 15_000 } + ); + // The admin's place is kept. + await expect(page.getByTestId('dm-filter-all')).toHaveClass(/is-on/); + await shot(page, 'r2-background-new-db'); +}); + +test('layout: 1024 rail, dark tabs, phone tabs and chips, phone access', async ({ + page, +}) => { + await page.setViewportSize({ width: 1024, height: 900 }); + await boot(page, {}, 'dark'); + await open(page); + await page.getByTestId('dm-filter-all').click(); + await page.waitForTimeout(500); + const overlaps = await page.evaluate( + () => + Array.from( + document.querySelectorAll('[data-testid="dm-rail-row"]') + ).filter(row => { + const strip = row.querySelector('df-sensitivity-strip .strip'); + const badge = row.querySelector('.rail-badge') as HTMLElement | null; + if (!strip || !badge || !badge.getBoundingClientRect().width) + return false; + const right = Math.max( + ...Array.from(strip.children).map( + t => (t as HTMLElement).getBoundingClientRect().right + ) + ); + return right > badge.getBoundingClientRect().left + 0.5; + }).length + ); + expect(overlaps).toBe(0); + await shot(page, 'r2-1024-rail-dark'); + const underline = await page + .locator('.dm-tabs .mdc-tab--active .mdc-tab-indicator__content--underline') + .evaluate(e => getComputedStyle(e).borderTopColor); + expect(underline).not.toBe('rgb(15, 7, 97)'); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.waitForTimeout(400); + // Phones: one select for the tabs (none hidden behind chevrons), and + // every filter chip in view. + await expect(page.getByTestId('dm-tab-columns')).toBeHidden(); + await expect(page.getByTestId('dm-tab-select')).toBeVisible(); + for (const f of ['review', 'masked', 'clear', 'all']) { + const box = await page.getByTestId(`dm-filter-${f}`).boundingBox(); + expect(box && box.x >= 0 && box.x + box.width <= 390).toBeTruthy(); + } + await shot(page, 'r2-390-columns-dark'); + + await page.setViewportSize({ width: 375, height: 812 }); + await page.getByTestId('dm-tab-select').click(); + await page.getByRole('option', { name: 'Access' }).click(); + await expect(page.getByTestId('dm-access')).toBeVisible(); + await page.waitForTimeout(400); + const spill = await page.evaluate(() => { + const list = document.querySelector('.admins') as HTMLElement | null; + return list ? list.scrollWidth - list.clientWidth : 0; + }); + expect(spill).toBeLessThanOrEqual(0); + await shot(page, 'r2-375-access-dark'); +}); + +test('masks: an often-numeric class shows what a number comes back as', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page); + await open(page, 'masks'); + const n = page.getByTestId('dm-numeric-example').first(); + await n.scrollIntoViewIfNeeded(); + await expect(n.locator('.mv-null')).toHaveText('null'); + await shot(page, 'r2-masks-numeric-example'); +}); + +/** + * Wide tables (RU-7): one service with 220 small tables and one wide + * table. Run against a production build; numbers land in perf-.json. + */ +for (const wide of [300, 1500]) { + test(`perf: a ${wide}-column table`, async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { uncovered: false }); + const cols: Record[] = []; + const mk = (t: string, c: string, i: number) => { + const v = i % 7 === 0 ? 'review' : i % 3 === 0 ? 'masked' : 'clear'; + return { + id: `big.${t}.${c}`, + service: 'big', + table: t, + column: c, + type: 'varchar', + verdict: v, + class: v === 'clear' ? null : 'email', + tier: v === 'clear' ? null : 'identity', + source: 'detected', + decided_by: null, + decided_at: null, + confidence: v === 'clear' ? 10 : 80, + example: v === 'clear' ? null : 'EMAIL_7f3a9c21', + format: v === 'clear' ? null : 'token', + evidence: { + total: 80, + threshold_mask: 70, + threshold_review: 40, + lines: [{ points: 80, kind: 'name', text: 'Column name' }], + shapes: [], + }, + }; + }; + for (let t = 0; t < 220; t++) { + for (let c = 0; c < 5; c++) { + cols.push(mk(`t${String(t).padStart(3, '0')}`, `c${c}`, t * 5 + c)); + } + } + for (let c = 0; c < wide; c++) cols.push(mk('a_wide', `col_${c}`, c)); + const counts: Record = { masked: 0, review: 0, clear: 0 }; + cols.forEach(c => counts[c['verdict'] as string]++); + const byTable = new Map[]>(); + cols.forEach(c => { + const k = c['table'] as string; + byTable.set(k, [...(byTable.get(k) ?? []), c]); + }); + const tables = [...byTable.entries()].map(([name, cs]) => { + const cc: Record = { masked: 0, review: 0, clear: 0 }; + cs.forEach(c => cc[c['verdict'] as string]++); + return { + name, + rows: null, + columns: cc, + strip: cs.map(c => ({ + column: c['column'], + mark: + c['verdict'] === 'review' + ? 'review' + : c['verdict'] === 'clear' + ? 'clear' + : 'identity', + tier: c['tier'], + })), + }; + }); + const now = new Date().toISOString(); + await page.route('**/api/v2/system/data_masking/table*', r => + r.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + resource: [ + { service: 'big', label: 'Big DB', scanned_at: now, tables }, + ], + }), + }) + ); + await page.route('**/api/v2/system/data_masking/column?*', r => + r.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ resource: cols }), + }) + ); + await page.route(/\/api\/v2\/system\/data_masking$/, async r => { + if (r.request().method() !== 'GET') return r.fallback(); + return r.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + you: { can_grant: true, sees_real_values: false }, + mode: 'preview', + scan: { + status: 'idle', + finished_at: now, + columns: cols.length, + last_status: 'done', + }, + counts, + coverage: { + covered: [{ service: 'big', label: 'Big DB', type: 'pgsql' }], + uncovered: [], + }, + }), + }); + }); + await open(page); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + const frame = () => + page.evaluate( + () => new Promise(r => requestAnimationFrame(() => setTimeout(r, 0))) + ); + const time = async ( + act: () => Promise, + done: () => Promise + ) => { + const t0 = Date.now(); + await act(); + await done(); + return Date.now() - t0; + }; + const filterAllMs = await time( + () => page.getByTestId('dm-filter-all').click(), + frame + ); + // Measured in the page: from the click to the frame where the first + // rows show, and to the frame where every row is drawn. + const opened = await page.evaluate(async n => { + const row = Array.from( + document.querySelectorAll('[data-testid="dm-rail-row"]') + ).find(e => e.textContent?.includes('a_wide'))!; + const t0 = performance.now(); + row.click(); + let first = 0; + await new Promise(res => { + const tick = () => { + const rows = document.querySelectorAll('[data-testid="dm-row"]'); + const title = document.querySelector('.list-title')?.textContent; + if (title === 'a_wide' && rows.length && !first) { + first = performance.now() - t0; + } + if (title === 'a_wide' && rows.length >= n) return res(); + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + }); + return { first, all: performance.now() - t0 }; + }, wide); + const openFirstMs = Math.round(opened.first); + const allRowsMs = Math.round(opened.all); + await expect(page.getByTestId('dm-row')).toHaveCount(wide); + // In-page timings (Playwright's own selector work on a large DOM is + // not the app's cost): from the event to the next painted frame. + const inPage = (what: 'key' | 'keep') => + page.evaluate(async w => { + const painted = () => + new Promise(r => requestAnimationFrame(() => setTimeout(r, 0))); + const t0 = performance.now(); + if (w === 'key') { + const input = document.querySelector( + '[data-testid="dm-search"]' + ) as HTMLInputElement; + input.value = 'c'; + input.dispatchEvent(new Event('input', { bubbles: true })); + await painted(); + const typed = performance.now() - t0; + // The rebuild runs once typing pauses (debounce). + await new Promise(r => setTimeout(r, 160)); + const t1 = performance.now(); + await painted(); + return { typed, rebuilt: performance.now() - t1 }; + } + ( + document.querySelector('[data-testid="dm-keep"]') as HTMLElement + ).click(); + await painted(); + return { typed: performance.now() - t0, rebuilt: 0 }; + }, what); + const key = await inPage('key'); + await page.getByTestId('dm-search').fill(''); + await page.waitForTimeout(400); + await page.getByTestId('dm-filter-review').click(); + await expect(page.getByTestId('dm-row')).toHaveCount(wide, { + timeout: 60_000, + }); + await frame(); + const keep = await inPage('keep'); + const keyMs = Math.round(key.typed + key.rebuilt); + const keepMs = Math.round(keep.typed); + const longTasks = await page.evaluate( + () => performance.getEntriesByType('longtask').length + ); + const out = { + wide, + columns: cols.length, + filterAllMs, + openFirstRowsMs: openFirstMs, + openAllRowsMs: allRowsMs, + keystrokeTypedMs: Math.round(key.typed), + keystrokeRebuildMs: Math.round(key.rebuilt), + keystrokeMs: keyMs, + keepMs, + longTasks, + }; + // eslint-disable-next-line no-console + console.log(JSON.stringify(out)); + const { writeFileSync } = await import('node:fs'); + writeFileSync(join(OUT, `perf-${wide}.json`), JSON.stringify(out, null, 2)); + }); +} + +/** WCAG contrast of an element's text on what is painted behind it. */ +async function contrastOf(page: Page, selector: string) { + return page + .locator(selector) + .first() + .evaluate(el => { + // rgb()/rgba(), or color(srgb r g b / a) for color-mix() results. + const parse = (c: string) => { + const m = c.match(/[\d.]+/g)!.map(Number); + const k = c.startsWith('color(') ? 255 : 1; + return { + r: m[0] * k, + g: m[1] * k, + b: m[2] * k, + a: m.length > 3 ? m[3] : 1, + }; + }; + const over = ( + top: ReturnType, + under: ReturnType + ) => ({ + r: top.r * top.a + under.r * (1 - top.a), + g: top.g * top.a + under.g * (1 - top.a), + b: top.b * top.a + under.b * (1 - top.a), + a: 1, + }); + // Composite the backgrounds from the page down to the element. + const chain: HTMLElement[] = []; + for ( + let e: HTMLElement | null = el as HTMLElement; + e; + e = e.parentElement + ) { + chain.unshift(e); + } + let bg = { r: 255, g: 255, b: 255, a: 1 }; + for (const e of chain) { + const c = parse(getComputedStyle(e).backgroundColor); + if (c.a > 0) bg = over(c, bg); + } + const fg = over(parse(getComputedStyle(el).color), bg); + const lum = (c: { r: number; g: number; b: number }) => { + const ch = [c.r, c.g, c.b].map(v => { + const s = v / 255; + return s <= 0.03928 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4; + }); + return 0.2126 * ch[0] + 0.7152 * ch[1] + 0.0722 * ch[2]; + }; + const [l1, l2] = [lum(fg), lum(bg)].sort((a, b) => b - a); + return Math.round(((l1 + 0.05) / (l2 + 0.05)) * 100) / 100; + }); +} + +for (const theme of ['light', 'dark'] as Theme[]) { + test(`contrast: small tier and "Not sensitive" text reach 4.5:1 (${theme})`, async ({ + page, + }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, {}, theme); + await open(page); + await page.getByTestId('dm-filter-all').click(); + await page.mouse.move(2, 890); + const clear = await contrastOf(page, '.chip.is-clear .chip-label'); + const identity = await contrastOf( + page, + '.mv.dm-tier-identity .mv-token, .mv.dm-tier-identity .mv-null' + ); + const chip = await contrastOf(page, '.chip.dm-tier-identity .chip-label'); + const badge = await contrastOf(page, '.rail-badge .badge'); + expect(clear).toBeGreaterThanOrEqual(4.5); + expect(identity).toBeGreaterThanOrEqual(4.5); + expect(chip).toBeGreaterThanOrEqual(4.5); + expect(badge).toBeGreaterThanOrEqual(4.5); + }); +} diff --git a/e2e/_data-masking-r3.spec.ts b/e2e/_data-masking-r3.spec.ts new file mode 100644 index 000000000..64bd04061 --- /dev/null +++ b/e2e/_data-masking-r3.spec.ts @@ -0,0 +1,375 @@ +/** + * Data Masking round-3 flows against the contract mock (exploratory: the + * leading underscore keeps it out of CI). Each test drives one fix and saves + * a screenshot of the screen it changed. Reads that are not Data Masking or + * Compliance (login, services, schemas) go to the live backend; no test + * writes to it. + * + * DM_SPEC=_data-masking-r3.spec.ts PLAYWRIGHT_BASE_URL=http://127.0.0.1:4321 \ + * DF_ADMIN_EMAIL=... DF_ADMIN_PASSWORD=... DM_SCREENS_DIR=... \ + * npx playwright test -c e2e/_data-masking-screens.config.ts + */ +import { Page, expect, test } from '@playwright/test'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { loginAsAdmin } from './fixtures/admin-login'; +import { + DataMaskingMockOptions, + installDataExplorerMaskingMock, + installDataMaskingMock, +} from './fixtures/data-masking-mock'; + +const OUT = process.env['DM_SCREENS_DIR'] ?? 'test-results/data-masking-r3'; +mkdirSync(OUT, { recursive: true }); +const BASE = '/dreamfactory/dist/#'; +const PAGE = `${BASE}/api-security/data-masking`; +type Theme = 'light' | 'dark'; + +test.describe.configure({ mode: 'default' }); +test.setTimeout(120_000); + +async function boot( + page: Page, + opts: DataMaskingMockOptions = {}, + theme: Theme = 'light' +) { + await page.addInitScript(t => { + localStorage.setItem('isDarkMode', JSON.stringify(t === 'dark')); + localStorage.setItem('isPhosphor', 'false'); + }, theme); + const state = await installDataMaskingMock(page, opts); + await loginAsAdmin(page); + return state; +} + +async function open(page: Page, tab = '') { + await page.goto(`${PAGE}${tab ? `?tab=${tab}` : ''}`); + await expect( + page + .locator( + '[data-testid="dm-mode"], [data-testid="dm-no-scan"], [data-testid="dm-no-databases"]' + ) + .first() + ).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId('dm-tab-columns')).toHaveText(/\S/); +} + +async function shot(page: Page, name: string, full = false) { + await page.addStyleTag({ + content: + '[id^="intercom"], .intercom-lightweight-app, iframe[name^="intercom"] { display: none !important; }', + }); + await page.waitForTimeout(400); + await page.screenshot({ + path: join(OUT, `${name}.png`), + caret: 'hide', + fullPage: full, + }); +} + +const log: Record = {}; +test.afterAll(() => + writeFileSync(join(OUT, 'r3-checks.json'), JSON.stringify(log, null, 2)) +); + +test('data explorer: masked filter matches no rows; masked sort ignored (decision 20)', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + const state = await boot(page, { mode: 'on' }); + await installDataExplorerMaskingMock(page, state); + await page.goto(`${BASE}/api-connections/data-explorer?service=crm`); + await page + .getByText('CustomerContacts', { exact: true }) + .first() + .click({ timeout: 60_000 }); + await expect(page.getByTestId('de-masked')).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId('de-masking-dropped')).toHaveCount(0); + + // Filter on a masked column: no rows, and the note says why. + const reads: string[] = []; + page.on('request', r => { + if (r.url().includes('/crm/_table/CustomerContacts')) reads.push(r.url()); + }); + const filterInput = page.locator('th.mat-column-filter_EMail .filter-input'); + await filterInput.fill('example'); + const note = page.getByTestId('de-masking-dropped'); + await expect(note).toHaveText(/Masked columns can't be filtered: EMail/, { + timeout: 15_000, + }); + await expect(page.locator('.footer-info').first()).toContainText('0 records'); + await expect(page.locator('.footer-info').first()).toContainText( + '(filtered)' + ); + const box = await note.boundingBox(); + log['de-filter'] = { + note: (await note.innerText()).trim(), + height: box?.height, + footer: (await page.locator('.footer-info').first().innerText()).trim(), + }; + await shot(page, 'r3-de-filter-masked'); + + // Sort by a masked column: rows come back, the sort is ignored. + await filterInput.fill(''); + await expect(note).toHaveCount(0, { timeout: 15_000 }); + await page.locator('th.mat-column-LastName').first().click(); + await expect(note).toHaveText(/Sort ignored on masked columns: LastName/, { + timeout: 15_000, + }); + log['de-sort'] = { note: (await note.innerText()).trim(), reads }; + await shot(page, 'r3-de-sort-masked'); +}); + +test('palette: pii, pci, phi and mask open with Data Masking', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page); + await page.goto(`${BASE}/home`); + await page.waitForTimeout(1500); + const found: Record = {}; + for (const q of ['pii', 'pci', 'phi', 'mask']) { + await page.keyboard.press('Control+k'); + const input = page.locator('.palette-search__input'); + await expect(input).toBeVisible(); + await input.fill(q); + await page.waitForTimeout(300); + const first = page.locator('.palette-option').first(); + await expect(first.locator('.palette-option__label')).toHaveText( + 'Data Masking' + ); + await expect(first).toHaveClass(/active/); + found[q] = await page + .locator('.palette-group__label, .palette-option__label') + .allInnerTexts(); + if (q === 'pii') await shot(page, 'r3-palette-pii'); + await page.keyboard.press('Escape'); + } + log['palette'] = found; +}); + +test('access: a stale write says the page shows the latest', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + const state = await boot(page); + await open(page, 'access'); + await expect(page.getByTestId('dm-access')).toBeVisible(); + // Someone else changes access (the CLI revokes every grant). + state.access.admins.grants = []; + state.accessVersion++; + await page.getByTestId('dm-role-3').locator('button').click(); + const toast = page.locator('mat-snack-bar-container').last(); + await expect(toast).toContainText( + 'Access settings changed elsewhere. Showing the latest.' + ); + await expect(toast).not.toContainText('Reload'); + await expect( + page.getByTestId('dm-admin-1').locator('input') + ).not.toBeChecked(); + log['stale'] = (await toast.innerText()).trim(); + await shot(page, 'r3-access-stale'); +}); + +test('access: duration options fit on one line next to "Exceeds HIPAA"', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page); + await page.route('**/api/v2/system/security_baseline', route => + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + resource: [{ id: 'hipaa', name: 'HIPAA', adopted: true }], + }), + }) + ); + await open(page, 'access'); + await page.locator('.admin.is-granted mat-select').first().click(); + await expect(page.getByTestId('dm-duration-exceeds').first()).toHaveText( + 'Exceeds HIPAA' + ); + // Every option is one text line tall. + const lines = await page.locator('mat-option').evaluateAll(opts => + opts.map(o => { + const text = o.querySelector( + '.mdc-list-item__primary-text' + ) as HTMLElement; + const lh = parseFloat(getComputedStyle(text).lineHeight) || 20; + return { + text: text.innerText.replace(/\s+/g, ' ').trim(), + lines: Math.round(text.getBoundingClientRect().height / lh), + }; + }) + ); + log['durations'] = lines; + expect(lines.length).toBe(5); + for (const l of lines) expect(l.lines, l.text).toBe(1); + await shot(page, 'r3-access-durations-hipaa'); + await page.keyboard.press('Escape'); + + // Phone: the wider list still fits the screen. + await page.setViewportSize({ width: 375, height: 800 }); + await page.waitForTimeout(400); + await page.locator('.admin.is-granted mat-select').first().click(); + await expect(page.getByTestId('dm-duration-exceeds').first()).toBeVisible(); + const panel = await page.locator('.mat-mdc-select-panel').evaluate(p => { + const r = p.getBoundingClientRect(); + return { left: r.left, right: r.right, vw: window.innerWidth }; + }); + log['durations375'] = panel; + expect(panel.left).toBeGreaterThanOrEqual(0); + expect(panel.right).toBeLessThanOrEqual(panel.vw); + await shot(page, 'r3-access-durations-375'); + await page.keyboard.press('Escape'); +}); + +test('columns: a computed field with no class is masked as Redact', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { computed: true }); + await open(page); + await page.getByTestId('dm-filter-all').click(); + await page.getByPlaceholder('Search tables and columns').fill('contact_line'); + const row = page.getByTestId('dm-row').filter({ hasText: 'contact_line' }); + await expect(row).toBeVisible(); + await expect(row.locator('.c-value')).toHaveText('Redact'); + await expect(row.getByTestId('dm-class-chip')).toContainText( + 'Choose a class' + ); + await row.getByTestId('dm-expand').click(); + await expect(page.locator('df-evidence-ledger')).toContainText( + 'Built from an expression that could not be checked' + ); + log['computed'] = (await row.innerText()).replace(/\s+/g, ' ').trim(); + await shot(page, 'r3-columns-computed-redact'); +}); + +test('a 403 on any write: the server says why, the screen stays as it was', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + const message = + 'Only a system administrator can change Data Masking settings.'; + const state = await boot(page, { mode: 'preview', forbidWrites: message }); + const toast = page.locator('mat-snack-bar-container').last(); + const out: Record = {}; + + // Mode. + await open(page); + await page.getByTestId('dm-mode-off').click(); + await expect(toast).toContainText(message); + await expect( + page.getByTestId('dm-mode').locator('.mat-button-toggle-checked') + ).toHaveText('Preview'); + out['mode'] = state.mode; + await shot(page, 'r3-forbidden-mode'); + + // A column decision. + await page.getByTestId('dm-filter-review').click(); + const firstRow = page.getByTestId('dm-row').first(); + const name = (await firstRow.locator('.c-name').innerText()).trim(); + const reviewBefore = state.columns.filter(c => c.verdict === 'review').length; + await firstRow.getByTestId('dm-clear').click(); + await expect(toast).toContainText(message); + await expect( + page.getByTestId('dm-row').filter({ hasText: name }).first() + ).toHaveClass(/is-review/); + out['review'] = [ + reviewBefore, + state.columns.filter(c => c.verdict === 'review').length, + ]; + + // Add a rule: the dialog stays open, nothing in place, the toast says why. + await open(page, 'detection'); + await page.getByRole('button', { name: 'Add rule' }).click(); + const dialog = page.getByRole('dialog'); + await dialog.locator('input').first().fill('^emp_ssn$'); + await dialog.getByRole('combobox').click(); + await page.getByRole('option', { name: 'National ID' }).click(); + await dialog + .getByRole('button', { name: /save|add/i }) + .last() + .click(); + await expect(toast).toContainText(message); + await expect(dialog).toBeVisible(); + await expect(dialog).not.toContainText(message); + await shot(page, 'r3-forbidden-rule'); + await page.keyboard.press('Escape'); + out['rules'] = state.rules.length; + + // Compliance Apply. + const cmsg = 'Only a system administrator can change Compliance baselines.'; + await page.goto(`${BASE}/api-security/compliance`); + await page + .getByRole('button', { name: /^Apply/ }) + .first() + .click({ timeout: 60_000 }); + await page + .getByRole('dialog') + .getByRole('button', { name: /^Apply/ }) + .click(); + await expect(toast).toContainText(cmsg); + out['compliance'] = (await toast.innerText()).trim(); + await shot(page, 'r3-forbidden-compliance-apply'); + log['forbidden'] = out; +}); + +test('GOLD without the package: the standard paywall, like every commercial feature', async ({ + page, +}) => { + // Paywall hits the shared component reports (the mock answers them). + const hits: string[] = []; + page.on('request', req => { + if ( + req.method() === 'POST' && + req.url().startsWith('https://updates.dreamfactory.com/api/paywall') + ) { + hits.push(String(req.postDataJSON()?.service_name)); + } + }); + await page.setViewportSize({ width: 1440, height: 900 }); + // GOLD license, no data_masking / security_baseline system resources. + await boot(page, { license: 'GOLD', installed: false }); + const out: Record = {}; + + for (const [path, name] of [ + ['data-masking', 'Data Masking'], + ['compliance', 'Compliance'], + ] as const) { + await page.goto(`${BASE}/api-security/${path}`); + const paywall = page.locator('df-paywall'); + await expect(paywall).toContainText('commercial', { timeout: 60_000 }); + await expect(page.getByTestId('feature-install')).toHaveCount(0); + await expect(page.locator('.dm, .cp')).toHaveCount(0); + await expect.poll(() => hits.includes(name)).toBe(true); + out[path] = (await paywall.locator('h2').first().innerText()).trim(); + await shot(page, `r3-paywall-gold-${path}`); + } + + // The Security overview card reads like the other paywalled cards: no + // status line. + await page.goto(`${BASE}/api-security`); + const card = page.locator('a.section-card[aria-label="Data Masking"]'); + await expect(card).toBeVisible({ timeout: 60_000 }); + await page.waitForTimeout(800); + await expect(card.locator('.section-card__meta')).toHaveCount(0); + await expect(page.getByText('Not installed')).toHaveCount(0); + await shot(page, 'r3-paywall-gold-security'); + + // Phone width: the same paywall, nothing scrolls sideways. + await page.setViewportSize({ width: 375, height: 800 }); + await page.goto(`${BASE}/api-security/data-masking`); + await expect(page.locator('df-paywall')).toContainText('commercial', { + timeout: 60_000, + }); + const m = await page.evaluate(() => ({ + doc: document.documentElement.scrollWidth, + viewport: document.documentElement.clientWidth, + })); + expect(m.doc).toBeLessThanOrEqual(m.viewport); + await shot(page, 'r3-paywall-gold-375'); + out['hits'] = hits; + log['goldWithoutPackage'] = out; +}); diff --git a/e2e/_data-masking-r4.spec.ts b/e2e/_data-masking-r4.spec.ts new file mode 100644 index 000000000..99c117fa1 --- /dev/null +++ b/e2e/_data-masking-r4.spec.ts @@ -0,0 +1,110 @@ +/** + * Data Masking round-4 flow against the contract mock (exploratory: the + * leading underscore keeps it out of CI): the server cannot read its scan + * history (overview `scan.unknown`, /table `scan_unknown`), so the screen + * says so instead of looking never-scanned, and offers no first scan. + * Login goes to the live backend; no test writes to it. + * + * DM_SPEC=_data-masking-r4.spec.ts PLAYWRIGHT_BASE_URL=http://127.0.0.1:4321 \ + * DF_ADMIN_EMAIL=... DF_ADMIN_PASSWORD=... DM_SCREENS_DIR=... \ + * npx playwright test -c e2e/_data-masking-screens.config.ts + */ +import { Page, expect, test } from '@playwright/test'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { loginAsAdmin } from './fixtures/admin-login'; +import { + DataMaskingMockOptions, + installDataMaskingMock, +} from './fixtures/data-masking-mock'; + +const OUT = process.env['DM_SCREENS_DIR'] ?? 'test-results/data-masking-r4'; +mkdirSync(OUT, { recursive: true }); +const PAGE = '/dreamfactory/dist/#/api-security/data-masking'; +const LINE = "Couldn't read the scan history. Check the DreamFactory log."; +type Theme = 'light' | 'dark'; + +test.describe.configure({ mode: 'default' }); +test.setTimeout(120_000); + +async function boot( + page: Page, + opts: DataMaskingMockOptions, + theme: Theme = 'light' +) { + await page.addInitScript(t => { + localStorage.setItem('isDarkMode', JSON.stringify(t === 'dark')); + localStorage.setItem('isPhosphor', 'false'); + }, theme); + await installDataMaskingMock(page, opts); + await loginAsAdmin(page); + await page.goto(PAGE); + await expect(page.getByTestId('dm-tab-columns')).toHaveText(/\S/, { + timeout: 60_000, + }); +} + +async function shot(page: Page, name: string) { + await page.addStyleTag({ + content: + '[id^="intercom"], .intercom-lightweight-app, iframe[name^="intercom"] { display: none !important; }', + }); + await page.waitForTimeout(400); + await page.screenshot({ path: join(OUT, `${name}.png`), caret: 'hide' }); +} + +const log: Record = {}; +test.afterAll(() => + writeFileSync(join(OUT, 'r4-checks.json'), JSON.stringify(log, null, 2)) +); + +for (const [w, theme] of [ + [1440, 'light'], + [375, 'dark'], +] as const) { + test(`unreadable scan history, nothing classified: one line, no first scan (${w} ${theme})`, async ({ + page, + }) => { + await page.setViewportSize({ width: w, height: 900 }); + await boot(page, { scenario: 'unscanned', scanUnknown: true }, theme); + const line = page.getByTestId('dm-scan-unknown'); + await expect(line).toContainText(LINE, { timeout: 30_000 }); + await expect(page.getByTestId('dm-no-scan')).toHaveCount(0); + await expect(page.getByTestId('dm-first-scan')).toHaveCount(0); + await expect(page.getByText('No scan yet')).toHaveCount(0); + await expect( + page.getByRole('button', { name: /scan databases|scan now/i }) + ).toHaveCount(0); + // The baseline shortcuts stay. + await expect(page.getByTestId('dm-first-run-baselines')).toBeVisible(); + log[`first-run-${w}`] = { + line: (await line.innerText()).trim(), + buttons: await page + .locator('.dm-panel button') + .evaluateAll(b => b.map(x => x.textContent?.trim())), + }; + await shot(page, `r4-scan-unknown-first-run-${w}-${theme}`); + }); +} + +test('unreadable scan history with classified columns: results stay, no "Not scanned yet"', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, { scanUnknown: true, pending: true }); + await expect(page.getByTestId('dm-search')).toBeVisible({ timeout: 30_000 }); + const line = page.getByTestId('dm-scan-unknown'); + await expect(line).toContainText(LINE); + await expect(page.getByTestId('dm-rail-unscanned')).toHaveCount(0); + await expect(page.getByText('Not scanned yet')).toHaveCount(0); + log['results'] = { + line: (await line.innerText()).trim(), + railGroups: await page + .locator('nav.rail > *') + .evaluateAll(els => els.map(e => e.getAttribute('data-testid'))), + }; + await shot(page, 'r4-scan-unknown-results-1440-light'); + // Other tabs keep working. + await page.getByTestId('dm-tab-access').click(); + await expect(page.getByTestId('dm-tab-access')).toHaveClass(/active/); +}); diff --git a/e2e/_data-masking-screens.config.ts b/e2e/_data-masking-screens.config.ts new file mode 100644 index 000000000..33d212f9c --- /dev/null +++ b/e2e/_data-masking-screens.config.ts @@ -0,0 +1,16 @@ +/** + * Runs only the Data Masking screenshot spec. The main config ignores `_*` + * specs (they never gate CI), so this one opts that single file back in. + * + * npx playwright test -c e2e/_data-masking-screens.config.ts + */ +import { defineConfig } from '@playwright/test'; +import base from '../playwright.config'; + +export default defineConfig({ + ...base, + testDir: '.', + testIgnore: [], + testMatch: [process.env['DM_SPEC'] ?? '_data-masking-screens.spec.ts'], + timeout: 180_000, +}); diff --git a/e2e/_data-masking-screens.spec.ts b/e2e/_data-masking-screens.spec.ts new file mode 100644 index 000000000..3641cdae5 --- /dev/null +++ b/e2e/_data-masking-screens.spec.ts @@ -0,0 +1,754 @@ +/** + * Data Masking and Compliance screenshots (exploratory: the leading + * underscore keeps it out of CI). Logs in against a live DreamFactory + * (any license), mocks /system/data_masking and /system/security_baseline + * per the df-masking contract with real classifier output, and saves PNGs. + * The real-backend journey lives in _data-masking-live.spec.ts. + * + * PLAYWRIGHT_BASE_URL=http://127.0.0.1:4300 \ + * DF_ADMIN_EMAIL=admin@dmui.local DF_ADMIN_PASSWORD=... \ + * DM_SCREENS_DIR=/path/to/screens/round1 \ + * npx playwright test e2e/_data-masking-screens.spec.ts + */ +import { Page, expect, test } from '@playwright/test'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { loginAsAdmin } from './fixtures/admin-login'; +import { + DataMaskingMockOptions, + installDataMaskingMock, +} from './fixtures/data-masking-mock'; + +const OUT = + process.env['DM_SCREENS_DIR'] ?? 'test-results/data-masking-screens'; +mkdirSync(OUT, { recursive: true }); + +const PAGE = '/dreamfactory/dist/#/api-security/data-masking'; +type Theme = 'light' | 'dark' | 'phosphor'; + +test.describe.configure({ mode: 'default' }); +test.setTimeout(180_000); + +async function boot( + page: Page, + theme: Theme, + opts: DataMaskingMockOptions = {} +) { + await page.addInitScript(t => { + localStorage.setItem('isDarkMode', JSON.stringify(t !== 'light')); + localStorage.setItem('isPhosphor', JSON.stringify(t === 'phosphor')); + }, theme); + await installDataMaskingMock(page, opts); + await loginAsAdmin(page); +} + +async function shot(page: Page, name: string, full = false) { + // Let motion and toasts settle; hide the caret so shots are stable. + await page.waitForTimeout(450); + await page.screenshot({ + path: join(OUT, `${name}.png`), + fullPage: full, + caret: 'hide', + }); +} + +async function open(page: Page, tab = '') { + await page.goto(`${PAGE}${tab ? `?tab=${tab}` : ''}`); + // The mode control waits for the first scan; first run leads with its + // call to action instead. + await expect( + page + .locator( + '[data-testid="dm-mode"], [data-testid="dm-no-scan"], [data-testid="dm-no-databases"]' + ) + .first() + ).toBeVisible({ timeout: 60_000 }); + // The dataMasking i18n scope loads lazily with the route. + await expect(page.getByTestId('dm-tab-columns')).toHaveText(/\S/, { + timeout: 20_000, + }); +} + +async function dismissToasts(page: Page) { + await page + .locator('mat-snack-bar-container') + .evaluateAll(els => + els.forEach(e => ((e as HTMLElement).style.display = 'none')) + ); +} + +for (const theme of ['light', 'dark'] as Theme[]) { + test(`data masking tour (${theme})`, async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, theme); + + await page.goto('/dreamfactory/dist/#/api-security'); + await expect(page.getByText('Manage data masking')).toBeVisible({ + timeout: 30_000, + }); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1200); + await shot(page, `security-overview-${theme}`); + + // Columns: review queue first. + await open(page); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + await shot(page, `columns-review-${theme}`); + + // Evidence ledger on a strong, multi-line verdict. + await page.getByTestId('dm-filter-masked').click(); + await page + .getByTestId('dm-rail-row') + .filter({ hasText: 'hr_staff' }) + .click(); + const row = page.getByTestId('dm-row').filter({ hasText: 'ni_number' }); + await row.getByTestId('dm-expand').click(); + const row2 = page.getByTestId('dm-row').filter({ hasText: 'full_name' }); + await row2.getByTestId('dm-expand').click(); + await shot(page, `columns-expanded-ledger-${theme}`); + + // Class picker menu. + await page + .getByTestId('dm-row') + .filter({ hasText: 'work_email' }) + .getByTestId('dm-class-chip') + .click(); + await expect(page.locator('.dm-class-menu')).toBeVisible(); + await shot(page, `columns-class-menu-${theme}`); + await page.keyboard.press('Escape'); + + // Review flow: decide every column in the first review table. + await page.getByTestId('dm-filter-review').click(); + await page + .getByTestId('dm-rail-row') + .filter({ hasText: 'facilities' }) + .click(); + const keeps = page.getByTestId('dm-keep'); + await keeps.first().click(); + await page.getByTestId('dm-clear').first().click(); + await shot(page, `columns-review-decided-${theme}`); + while ((await page.getByTestId('dm-keep').count()) > 0) { + await page.getByTestId('dm-keep').first().click(); + } + await expect(page.getByTestId('dm-next-table')).toBeVisible(); + await dismissToasts(page); + await shot(page, `columns-next-table-${theme}`); + + // Access. + await page.getByTestId('dm-tab-access').click(); + await expect(page.getByTestId('dm-access')).toBeVisible(); + await page.getByTestId('dm-admin-2').locator('input').check(); + await page.getByTestId('dm-role-4').locator('button').click(); + await page.waitForTimeout(300); + await dismissToasts(page); + await shot(page, `access-${theme}`); + + // Masks: switch email to first letter so the example re-renders. + await page.getByTestId('dm-tab-masks').click(); + await expect(page.getByTestId('dm-masks')).toBeVisible(); + await page.getByTestId('dm-format-email').click(); + await page.getByRole('option', { name: 'First letter' }).click(); + await dismissToasts(page); + await shot(page, `masks-${theme}`); + + // Detection + add rule dialog with a live preview. + await page.getByTestId('dm-tab-detection').click(); + await expect(page.getByTestId('dm-detection')).toBeVisible(); + await page.getByTestId('dm-builtin').click(); + await shot(page, `detection-${theme}`); + await page.getByTestId('dm-builtin').click(); + await page.getByTestId('dm-add-rule').click(); + await page + .getByTestId('dm-rule-pattern') + .fill('(^|_)(lat|lng|latitude|longitude)$'); + await page.getByTestId('dm-rule-class').click(); + await page.getByRole('option', { name: 'Precise location' }).click(); + await expect(page.getByTestId('dm-rule-preview')).toContainText('Matches'); + await page.waitForTimeout(500); + await shot(page, `detection-add-rule-dialog-${theme}`); + await page.getByTestId('dm-rule-save').click(); + await dismissToasts(page); + + // Activity. + await page.getByTestId('dm-tab-activity').click(); + await expect(page.getByTestId('dm-activity')).toBeVisible(); + await page.waitForTimeout(300); + await shot(page, `activity-${theme}`); + + // Mode: On asks first, with the real numbers. + await page.getByTestId('dm-mode-on').click(); + await expect(page.getByTestId('dm-confirm-ok')).toBeVisible(); + await shot(page, `mode-on-confirm-${theme}`); + await page.getByTestId('dm-confirm-ok').click(); + await page.getByTestId('dm-tab-columns').click(); + await page.waitForTimeout(400); + await shot(page, `mode-on-toast-${theme}`); + + // Off asks too; the toggle keeps showing On until it is confirmed. + await dismissToasts(page); + await page.getByTestId('dm-mode-off').click(); + await expect(page.getByTestId('dm-confirm-cancel')).toBeVisible(); + await shot(page, `mode-off-confirm-${theme}`); + await page.getByTestId('dm-confirm-cancel').click(); + }); +} + +test('scan progress and wide-table strips', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light'); + await open(page); + await page.getByTestId('dm-scan').click(); + await expect(page.getByTestId('dm-scan')).toHaveText(/Scanning [1-9]/, { + timeout: 20_000, + }); + await page.screenshot({ path: join(OUT, 'scanning-light.png') }); + await expect(page.getByTestId('dm-scan')).toHaveText(/Scan now/, { + timeout: 30_000, + }); + await dismissToasts(page); + await page.getByTestId('dm-filter-all').click(); + await page.getByTestId('dm-search').fill('american_football'); + await page.waitForTimeout(300); + await page.getByTestId('dm-rail-row').first().hover(); + await page.locator('df-sensitivity-strip .strip').first().hover(); + await page.waitForTimeout(900); + await shot(page, 'strips-wide-tooltip-light'); + await page.getByTestId('dm-search').fill(''); + await page.getByTestId('dm-filter-masked').click(); + await page.waitForTimeout(300); + await shot(page, 'columns-masked-light'); +}); + +test('phosphor masks and strips', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'phosphor'); + await open(page, 'masks'); + await expect(page.getByTestId('dm-masks')).toBeVisible(); + await shot(page, 'masks-phosphor'); + await page.getByTestId('dm-tab-columns').click(); + await page.getByTestId('dm-filter-all').click(); + await shot(page, 'columns-all-phosphor'); +}); + +test('narrow 1024 dark access and masked', async ({ page }) => { + await page.setViewportSize({ width: 1024, height: 768 }); + await boot(page, 'dark'); + await open(page); + await page.getByTestId('dm-filter-masked').click(); + await shot(page, 'columns-1024-masked-dark'); + await page.getByTestId('dm-tab-access').click(); + await expect(page.getByTestId('dm-access')).toBeVisible(); + await page.getByTestId('dm-admin-mode').getByText('Selected').click(); + await page.waitForTimeout(400); + await dismissToasts(page); + await shot(page, 'access-1024-dark'); +}); + +test('phosphor columns', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'phosphor'); + await open(page); + await page.getByTestId('dm-filter-masked').click(); + await page.getByTestId('dm-rail-row').filter({ hasText: 'hr_staff' }).click(); + await page + .getByTestId('dm-row') + .filter({ hasText: 'ni_number' }) + .getByTestId('dm-expand') + .click(); + await shot(page, 'columns-phosphor'); +}); + +test('mobile columns', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await boot(page, 'light'); + await open(page); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + await shot(page, 'columns-mobile-light'); +}); + +test('narrow 1024 columns', async ({ page }) => { + await page.setViewportSize({ width: 1024, height: 768 }); + await boot(page, 'light'); + await open(page); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + await shot(page, 'columns-1024-light'); +}); + +test('states: never scanned, all reviewed, no grant rights', async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light', { scenario: 'unscanned' }); + await open(page); + await expect(page.getByTestId('dm-no-scan')).toBeVisible(); + await shot(page, 'columns-no-scan-light'); + // Nothing classified yet and masking off: the page leads with the scan. + await expect(page.getByTestId('dm-mode')).toHaveCount(0); +}); + +test('state: all reviewed', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light', { scenario: 'reviewed' }); + await open(page); + // Nothing left to review: no To review chip (the header hides it too), + // and the page opens on Masked. + await expect(page.getByTestId('dm-filter-review')).toHaveCount(0); + await expect(page.getByTestId('dm-count-review')).toHaveCount(0); + await expect(page.getByTestId('dm-filter-masked')).toHaveClass(/is-on/); + await shot(page, 'columns-all-reviewed-light'); +}); + +test('state: no grant rights', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light', { scenario: 'nogrant' }); + await open(page, 'access'); + await expect(page.getByTestId('dm-access-lock')).toBeVisible(); + await shot(page, 'access-locked-light'); +}); + +test('paywall on open source', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light', { license: 'OPEN SOURCE' }); + await page.goto(PAGE); + await expect(page.locator('df-paywall')).toContainText('commercial', { + timeout: 20_000, + }); + await shot(page, 'paywall-opensource'); +}); + +/* + * Final product-owner set: numbered, clean full-viewport captures. + * DM_SCREENS_DIR=.../screens/final npx playwright test \ + * -c e2e/_data-masking-screens.config.ts -g "final set" + * Each test also writes .console.json with console errors and any + * raw transloco keys found in the visible text. + */ +const RAW_KEY = + /\b(dataMasking|common|nav|apiSecurity|security)\.[a-zA-Z][\w.]+/g; + +function watchConsole(page: Page) { + const errors: string[] = []; + page.on('console', m => { + if (m.type() === 'error') errors.push(m.text().slice(0, 400)); + }); + page.on('pageerror', e => errors.push(`pageerror: ${e.message}`)); + page.on('response', r => { + if (r.status() >= 400) + errors.push(`http ${r.status()} ${r.request().method()} ${r.url()}`); + }); + return errors; +} + +async function finalShot(page: Page, name: string, errors: string[]) { + // Hide the third-party Intercom launcher; it is not part of the feature. + await page.addStyleTag({ + content: + '[id^="intercom"], .intercom-lightweight-app, iframe[name^="intercom"] { display: none !important; }', + }); + await page.mouse.move(2, 890); + await page.waitForTimeout(600); + const text = await page.evaluate(() => document.body.innerText); + const keys = Array.from(new Set(text.match(RAW_KEY) ?? [])); + writeFileSync( + join(OUT, `${name}.console.json`), + JSON.stringify({ errors: [...errors], rawKeys: keys }, null, 2) + ); + await page.screenshot({ path: join(OUT, `${name}.png`), caret: 'hide' }); +} + +test('final set (light)', async ({ page }) => { + const errors = watchConsole(page); + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light'); + + // Reach the overview in-app once the root translations have loaded: a cold + // deep link renders its actions/notes as raw keys (see report). + await page.waitForLoadState('networkidle'); + await expect(page.getByRole('link', { name: 'Home' }).first()).toBeVisible({ + timeout: 60_000, + }); + await page.waitForTimeout(1500); + await page.evaluate(() => (window.location.hash = '#/api-security')); + await expect(page.getByText('Manage data masking')).toBeVisible({ + timeout: 60_000, + }); + await expect(page.getByText(/provider types available/i)).toBeVisible({ + timeout: 60_000, + }); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1200); + const overlays = await page.evaluate(() => + Array.from( + document.querySelectorAll( + '.cdk-overlay-backdrop, .mat-drawer-backdrop.mat-drawer-shown, .cdk-overlay-pane' + ) + ).map( + e => `${e.className} :: ${(e as HTMLElement).innerText?.slice(0, 120)}` + ) + ); + errors.push(...overlays.map(o => `overlay: ${o}`)); + await finalShot(page, '01-security-overview', errors); + + await open(page); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + await expect(page.getByTestId('dm-filter-review')).toBeVisible(); + await finalShot(page, '02-columns-review-light', errors); + + await page.getByTestId('dm-filter-masked').click(); + await page.getByTestId('dm-rail-row').filter({ hasText: 'hr_staff' }).click(); + await page + .getByTestId('dm-row') + .filter({ hasText: 'ni_number' }) + .getByTestId('dm-expand') + .click(); + await page + .getByTestId('dm-row') + .filter({ hasText: 'full_name' }) + .getByTestId('dm-expand') + .click(); + await finalShot(page, '03-columns-ledger-expanded-light', errors); + + // Collapse the ledgers so the menu has the table behind it. + for (const c of ['ni_number', 'full_name']) { + await page + .getByTestId('dm-row') + .filter({ hasText: c }) + .getByTestId('dm-expand') + .click(); + } + await page + .getByTestId('dm-row') + .filter({ hasText: 'work_email' }) + .getByTestId('dm-class-chip') + .click(); + await expect(page.locator('.dm-class-menu')).toBeVisible(); + await page.waitForTimeout(400); + await page.screenshot({ + path: join(OUT, '05-class-picker-open.png'), + caret: 'hide', + }); + await page.keyboard.press('Escape'); + + await page.getByTestId('dm-tab-access').click(); + await expect(page.getByTestId('dm-access')).toBeVisible(); + await page.waitForTimeout(500); + await dismissToasts(page); + await finalShot(page, '07-access-light', errors); + + await page.getByTestId('dm-tab-masks').click(); + await expect(page.getByTestId('dm-masks')).toBeVisible(); + await page.waitForTimeout(500); + await finalShot(page, '08-masks-light', errors); + + await page.getByTestId('dm-tab-detection').click(); + await expect(page.getByTestId('dm-detection')).toBeVisible(); + await page.getByTestId('dm-add-rule').click(); + await page + .getByTestId('dm-rule-pattern') + .fill('(^|_)(lat|lng|latitude|longitude)$'); + await page.getByTestId('dm-rule-class').click(); + await page.getByRole('option', { name: 'Precise location' }).click(); + await expect(page.getByTestId('dm-rule-preview')).toContainText('Matches'); + await page.waitForTimeout(600); + await finalShot(page, '09-detection-add-rule', errors); + await page.keyboard.press('Escape'); + + await page.getByTestId('dm-tab-activity').click(); + await expect(page.getByTestId('dm-activity')).toBeVisible(); + await page.waitForTimeout(800); + await finalShot(page, '10-activity-light', errors); + + await page.getByTestId('dm-tab-columns').click(); + await page.getByTestId('dm-mode-on').click(); + await expect(page.getByTestId('dm-confirm-ok')).toBeVisible(); + await page.waitForTimeout(500); + await finalShot(page, '06-mode-on-confirm', errors); + await page.getByTestId('dm-confirm-cancel').click(); +}); + +test('final set (dark)', async ({ page }) => { + const errors = watchConsole(page); + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'dark'); + await open(page); + await page.getByTestId('dm-filter-masked').click(); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + await page.waitForTimeout(400); + await finalShot(page, '04-columns-masked-dark', errors); + + await page.getByTestId('dm-tab-access').click(); + await expect(page.getByTestId('dm-access')).toBeVisible(); + await page.waitForTimeout(500); + await finalShot(page, '12-access-dark', errors); +}); + +test('final set (mobile)', async ({ page }) => { + const errors = watchConsole(page); + await page.setViewportSize({ width: 390, height: 844 }); + await boot(page, 'light'); + await open(page); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + await page.waitForTimeout(400); + await finalShot(page, '11-columns-mobile-390', errors); +}); + +/* + * Build 2 set: first run (baselines row, first-scan progress, Preview after + * the first scan), coverage in the Columns rail, and the Compliance page, + * in light and dark at 1440 and 390, plus phosphor / 1024 / paywall checks. + * DM_SCREENS_DIR=.../screens/build2 npx playwright test \ + * -c e2e/_data-masking-screens.config.ts -g "build2" + */ +const COMPLIANCE = '/dreamfactory/dist/#/api-security/compliance'; + +async function openCompliance(page: Page, baseline = '') { + await page.goto(`${COMPLIANCE}${baseline ? `?baseline=${baseline}` : ''}`); + await expect(page.getByTestId('cp-detail')).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId('cp-controls')).toBeVisible({ + timeout: 20_000, + }); +} + +async function hideChrome(page: Page) { + await page.addStyleTag({ + content: + '[id^="intercom"], .intercom-lightweight-app, iframe[name^="intercom"] { display: none !important; }', + }); +} + +async function shot2(page: Page, name: string, full = false) { + await hideChrome(page); + await page.mouse.move(2, 2); + await page.waitForTimeout(500); + await page.screenshot({ + path: join(OUT, `${name}.png`), + fullPage: full, + caret: 'hide', + }); +} + +test.describe('build2', () => { + for (const theme of ['light', 'dark'] as Theme[]) { + for (const width of [1440, 390]) { + const size = { width, height: width === 390 ? 844 : 900 }; + const tag = `${theme}-${width}`; + + test(`build2 first run ${tag}`, async ({ page }) => { + await page.setViewportSize(size); + await boot(page, theme, { scenario: 'unscanned' }); + await open(page); + await expect(page.getByTestId('dm-no-scan')).toBeVisible(); + await expect(page.getByTestId('dm-first-run-baselines')).toBeVisible(); + await expect(page.getByTestId('dm-mode')).toHaveCount(0); + await shot2(page, `first-run-${tag}`); + + await page.getByTestId('empty-state-action').click(); + await expect(page.getByTestId('dm-first-scan')).toContainText( + /Scanning [1-9]/, + { timeout: 20_000 } + ); + await shot2(page, `first-scan-progress-${tag}`); + + // The server switched Off -> Preview; the page follows, no reload. + await expect(page.getByTestId('dm-mode-preview')).toHaveClass( + /mat-button-toggle-checked/, + { timeout: 30_000 } + ); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + await expect( + page.getByText('Scan finished. Preview is on.') + ).toBeVisible(); + await shot2(page, `first-scan-done-preview-${tag}`); + }); + + test(`build2 columns coverage ${tag}`, async ({ page }) => { + await page.setViewportSize(size); + await boot(page, theme, { pending: true }); + await open(page); + await page.getByTestId('dm-filter-masked').click(); + await expect(page.getByTestId('dm-row').first()).toBeVisible(); + if (width === 1440) { + const uncovered = page.getByTestId('dm-rail-uncovered'); + await uncovered.scrollIntoViewIfNeeded(); + await expect(uncovered).toContainText('Event store'); + await expect(page.getByTestId('dm-rail-unscanned')).toContainText( + 'HR (SQL Server)' + ); + await page.getByTestId('dm-rail-uncovered-row').first().hover(); + await page.waitForTimeout(900); + await hideChrome(page); + await page.screenshot({ + path: join(OUT, `columns-coverage-${tag}.png`), + caret: 'hide', + }); + } else { + await page.locator('.table-select mat-select').click(); + const notCovered = page + .locator('mat-optgroup') + .filter({ hasText: 'Not covered' }); + await expect(notCovered).toBeVisible(); + await notCovered.scrollIntoViewIfNeeded(); + await page.waitForTimeout(400); + await hideChrome(page); + await page.screenshot({ + path: join(OUT, `columns-coverage-${tag}.png`), + caret: 'hide', + }); + await page.keyboard.press('Escape'); + await shot2(page, `columns-masked-${tag}`); + } + }); + + test(`build2 compliance ${tag}`, async ({ page }) => { + await page.setViewportSize(size); + await boot(page, theme); + await openCompliance(page, 'hipaa'); + await expect(page.locator('#cp-title')).toHaveText('HIPAA baseline'); + await shot2(page, `compliance-hipaa-${tag}`); + if (width === 1440) + await shot2(page, `compliance-hipaa-full-${tag}`, true); + + // Apply: the dialog lists exactly what will change. + await page.getByTestId('cp-apply').click(); + await expect(page.getByTestId('cp-apply-changes')).toBeVisible(); + await shot2(page, `compliance-apply-dialog-${tag}`); + await page.getByTestId('cp-apply-turn-on').locator('input').check(); + await page.getByTestId('cp-apply-confirm').click(); + await expect(page.getByTestId('cp-tracking')).toBeVisible(); + await expect( + page.getByText('HIPAA baseline applied. Masking is on.') + ).toBeVisible(); + await shot2(page, `compliance-applied-${tag}`); + await dismissToasts(page); + + // Manual checks expand in place. + await page.getByTestId('cp-manual-toggle').click(); + await page + .locator('.controls.is-manual .control') + .last() + .scrollIntoViewIfNeeded(); + await shot2(page, `compliance-manual-${tag}`); + + // Another vertical. + if (width === 1440) { + await page.getByTestId('cp-rail-pci_dss').click(); + } else { + await page.getByTestId('cp-select').click(); + await page.getByRole('option', { name: /PCI DSS/ }).click(); + } + await expect(page.locator('#cp-title')).toHaveText('PCI DSS baseline'); + await page.waitForTimeout(300); + await shot2(page, `compliance-pci-${tag}`); + + // Export: a real download of the evidence report. + await page.getByTestId('cp-export').click(); + const dl = page.waitForEvent('download'); + await page.getByTestId('cp-export-csv').click(); + const file = await dl; + expect(file.suggestedFilename()).toMatch( + /^dreamfactory-pci_dss-baseline-\d{4}-\d{2}-\d{2}\.csv$/ + ); + }); + } + } + + test('build2 first run with no SQL database', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light', { scenario: 'nodb' }); + await open(page); + await expect(page.getByTestId('dm-no-databases')).toBeVisible(); + await expect(page.getByTestId('dm-first-run-baselines')).toBeVisible(); + await shot2(page, 'first-run-no-databases-light-1440'); + await page.getByTestId('empty-state-action').click(); + await expect(page).toHaveURL(/api-types\/database\/create/); + }); + + test('build2 baseline chip opens compliance on it', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light', { scenario: 'unscanned' }); + await open(page); + await page.getByTestId('dm-baseline-pci_dss').click(); + await expect(page).toHaveURL(/compliance\?baseline=pci_dss/); + await expect(page.locator('#cp-title')).toHaveText('PCI DSS baseline', { + timeout: 30_000, + }); + // Nothing scanned yet: the scan check fails and links back. + await expect( + page.getByTestId('cp-control-masking.scanned').getByTestId('cp-fix') + ).toBeVisible(); + await shot2(page, 'compliance-from-first-run-light-1440'); + }); + + test('build2 security overview with compliance', async ({ page }) => { + for (const theme of ['light', 'dark'] as Theme[]) { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, theme); + await page.goto('/dreamfactory/dist/#/api-security'); + await expect(page.getByText('Manage compliance')).toBeVisible({ + timeout: 60_000, + }); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1200); + await shot2(page, `security-overview-${theme}-1440`); + await page.context().clearCookies(); + } + }); + + test('build2 compliance phosphor and 1024', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'phosphor'); + await openCompliance(page, 'pci_dss'); + await shot2(page, 'compliance-pci-phosphor-1440'); + await page.setViewportSize({ width: 1024, height: 768 }); + await page.waitForTimeout(400); + await shot2(page, 'compliance-pci-phosphor-1024'); + }); + + test('build2 compliance 1024 light', async ({ page }) => { + await page.setViewportSize({ width: 1024, height: 768 }); + await boot(page, 'light'); + await openCompliance(page, 'soc2'); + await shot2(page, 'compliance-soc2-light-1024'); + }); + + test('build2 compliance paywall on open source', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light', { license: 'OPEN SOURCE' }); + await page.goto(COMPLIANCE); + await expect(page.locator('df-paywall')).toContainText('commercial', { + timeout: 30_000, + }); + await shot2(page, 'compliance-paywall-opensource'); + }); + + test('build2 data masking glyph bars and tokens', async ({ page }) => { + await page.setViewportSize({ width: 1440, height: 900 }); + await boot(page, 'light'); + await open(page); + await page.getByTestId('dm-filter-masked').click(); + await page + .getByTestId('dm-rail-row') + .filter({ hasText: 'hr_staff' }) + .click(); + await page + .getByTestId('dm-row') + .filter({ hasText: 'ni_number' }) + .getByTestId('dm-expand') + .click(); + // No raw glyphs on screen: every mask run is drawn as a bar. + const text = await page.locator('df-dm-columns').innerText(); + expect(text).not.toContain('•'); + expect(await page.locator('.mv-bar').count()).toBeGreaterThan(3); + await expect(page.locator('.mv-token').first()).toHaveText( + /^[A-Z]+_[0-9a-f]{8}$/ + ); + await shot2(page, 'columns-masked-ledger-light-1440'); + await page.getByTestId('dm-tab-masks').click(); + await expect(page.getByTestId('dm-masks')).toBeVisible(); + await shot2(page, 'masks-light-1440'); + await page.getByTestId('dm-tab-activity').click(); + await expect(page.getByTestId('dm-activity')).toBeVisible(); + await page.waitForTimeout(500); + await shot2(page, 'activity-light-1440'); + }); +}); diff --git a/e2e/fixtures/compliance-mock.ts b/e2e/fixtures/compliance-mock.ts new file mode 100644 index 000000000..0d4d220ea --- /dev/null +++ b/e2e/fixtures/compliance-mock.ts @@ -0,0 +1,731 @@ +/** + * Playwright route mock for /api/v2/system/security_baseline/** (Compliance + * baselines), stateful on top of the Data Masking mock: checks are computed + * from the masking state (mode, review queue, formats, access), Apply + * changes exactly what its `changes` list says, and every write lands in the + * masking Activity. Platform checks (HTTPS, CORS, session TTL, ...) are + * fixed per mock instance. Reports carry no data values. + * + * Installed by installDataMaskingMock(); not meant to be used on its own. + */ +import { Page, Route } from '@playwright/test'; + +type Mode = 'off' | 'preview' | 'on'; + +/** The slice of the masking mock state the baselines read and write. */ +export interface MaskingStateView { + mode: Mode; + /** Refuse every write with 403 and this message (null: allow). */ + forbidWrites?: string | null; + /** The masking mock's scan; status/progress drive the first scan. */ + scan: { + finished_at: string | null; + status?: 'idle' | 'running'; + progress?: number; + }; + columns: { verdict: string; class: string | null; source: string }[]; + masks: { class: string; format: string; formats_allowed: string[] }[]; + access: { + admins: { + mode: 'none' | 'selected' | 'all'; + grants: { admin_id: number; name: string; expires_at: string | null }[]; + }; + ai: string; + roles: { role_id: number; name: string; enabled: boolean; scope: string }[]; + }; + rules: { + id: number; + match: 'name' | 'value'; + pattern: string; + class: string; + }[]; + nextRule: number; + ev: ( + msAgo: number, + actor: string, + kind: string, + text: string, + target?: string + ) => void; +} + +interface Check { + id: string; + refs: string[]; + title?: string; +} + +interface BaselineDef { + id: string; + name: string; + vertical: string; + frameworks: string[]; + /** Longest admin grant allowed, in hours. */ + grantMax: number; + /** Per-class formats allowed by this baseline. */ + formats: { class: string; allowed: string[]; title: string }[]; + rolesRestricted: boolean; + adminsAllForbidden: boolean; + checks: Check[]; + manual: { id: string; title: string; refs: string[] }[]; + rules: { match: 'name' | 'value'; pattern: string; class: string }[]; +} + +const HOUR = 3_600_000; +const ME = 'Kevin Admin'; + +// Framework references: only ones double-checked against the source texts +// (HIPAA 45 CFR Part 164, PCI DSS v4.0, GLBA Safeguards Rule 16 CFR 314, +// CJIS Security Policy, GDPR, AICPA TSC 2017, ISO/IEC 27001:2022 Annex A). +const BASELINES: BaselineDef[] = [ + { + id: 'hipaa', + name: 'HIPAA', + vertical: 'healthcare', + frameworks: ['HIPAA Security Rule', 'HIPAA Privacy Rule'], + grantMax: 8, + formats: [ + { + class: 'date_of_birth', + allowed: ['year', 'redact', 'null'], + title: 'Birth dates show at most the year', + }, + ], + rolesRestricted: true, + adminsAllForbidden: true, + checks: [ + { id: 'masking.mode', refs: ['164.312(a)(1)', '164.514(b)'] }, + { id: 'masking.scanned', refs: ['164.514(b)'] }, + { id: 'masking.review', refs: ['164.514(b)'] }, + { id: 'masking.format', refs: ['164.514(b)'] }, + { id: 'access.ai', refs: ['164.312(a)(1)'] }, + { id: 'access.grant_expiry', refs: ['164.312(a)(1)'] }, + { id: 'access.admins_all', refs: ['164.312(a)(1)'] }, + { id: 'access.roles_restricted', refs: ['164.312(a)(1)'] }, + { id: 'platform.https', refs: ['164.312(e)(1)'] }, + { id: 'platform.session_ttl', refs: ['164.312(a)(2)(iii)'] }, + { id: 'platform.audit', refs: ['164.312(b)'] }, + { id: 'platform.root_admin', refs: ['164.312(a)(1)'] }, + ], + manual: [ + { + id: 'manual.encryption_at_rest', + title: 'Encryption at rest on your databases', + refs: ['164.312(a)(2)(iv)'], + }, + { + id: 'manual.baa', + title: 'Business associate agreements in place', + refs: ['164.308(b)(1)'], + }, + ], + rules: [ + { + match: 'name', + pattern: '(^|_)(mrn|medical_record|diagnosis|icd10?)($|_)', + class: 'special_category', + }, + ], + }, + { + id: 'pci_dss', + name: 'PCI DSS', + vertical: 'payments', + frameworks: ['PCI DSS v4.0.1'], + grantMax: 8, + formats: [ + { + class: 'card_number', + allowed: ['last4', 'redact', 'null'], + title: 'Card numbers show at most the last 4 digits', + }, + { + class: 'card_security', + allowed: ['null', 'redact'], + title: 'Card security codes never shown', + }, + ], + rolesRestricted: true, + adminsAllForbidden: true, + checks: [ + { id: 'masking.mode', refs: ['3.4.1'] }, + { id: 'masking.scanned', refs: ['3.4.1'] }, + { id: 'masking.review', refs: ['3.4.1'] }, + { id: 'masking.format', refs: ['3.3.1', '3.4.1'] }, + { id: 'access.ai', refs: ['7.2.1'] }, + { id: 'access.grant_expiry', refs: ['7.2.1'] }, + { id: 'access.admins_all', refs: ['7.2.1'] }, + { id: 'access.roles_restricted', refs: ['7.2.1'] }, + { id: 'platform.https', refs: ['4.2.1'] }, + { id: 'platform.session_ttl', refs: ['8.2.8'] }, + { id: 'platform.audit', refs: ['10.2.1'] }, + { id: 'platform.rate_limit', refs: [] }, + ], + manual: [ + { + id: 'manual.encryption_at_rest', + title: 'Stored card numbers are unreadable at rest', + refs: ['3.5.1'], + }, + ], + rules: [ + { + match: 'name', + pattern: '(^|_)(pan|cc_?num|card_?no)($|_)', + class: 'card_number', + }, + { + match: 'name', + pattern: '(^|_)(cvv2?|cvc|csc)($|_)', + class: 'card_security', + }, + ], + }, + { + id: 'glba', + name: 'GLBA', + vertical: 'banking', + frameworks: ['GLBA Safeguards Rule (16 CFR 314)'], + grantMax: 24, + formats: [], + rolesRestricted: true, + adminsAllForbidden: false, + checks: [ + { id: 'masking.mode', refs: ['314.4(c)(1)'] }, + { id: 'masking.scanned', refs: ['314.4(c)(1)'] }, + { id: 'masking.review', refs: ['314.4(c)(1)'] }, + { id: 'access.ai', refs: ['314.4(c)(1)'] }, + { id: 'access.grant_expiry', refs: ['314.4(c)(1)'] }, + { id: 'access.roles_restricted', refs: ['314.4(c)(1)'] }, + { id: 'platform.https', refs: ['314.4(c)(3)'] }, + { id: 'platform.audit', refs: ['314.4(c)(8)'] }, + ], + manual: [ + { + id: 'manual.encryption_at_rest', + title: 'Customer information encrypted at rest', + refs: ['314.4(c)(3)'], + }, + ], + rules: [], + }, + { + id: 'cjis', + name: 'CJIS', + vertical: 'government', + frameworks: ['FBI CJIS Security Policy v5.9'], + grantMax: 8, + formats: [], + rolesRestricted: true, + adminsAllForbidden: true, + checks: [ + { id: 'masking.mode', refs: ['5.5.2'] }, + { id: 'masking.review', refs: ['5.5.2'] }, + { id: 'access.ai', refs: ['5.5.2'] }, + { id: 'access.grant_expiry', refs: ['5.5.2'] }, + { id: 'access.admins_all', refs: ['5.5.2'] }, + { id: 'access.roles_restricted', refs: ['5.5.2'] }, + { id: 'platform.session_ttl', refs: ['5.5.5'] }, + { id: 'platform.https', refs: ['5.10.1.2'] }, + { id: 'platform.audit', refs: ['5.4'] }, + ], + manual: [ + { + id: 'manual.encryption_at_rest', + title: 'Criminal justice information encrypted at rest', + refs: ['5.10.1.2'], + }, + ], + rules: [], + }, + { + id: 'gdpr', + name: 'GDPR', + vertical: 'privacy', + frameworks: ['EU GDPR (Regulation (EU) 2016/679)'], + grantMax: 24, + formats: [], + rolesRestricted: false, + adminsAllForbidden: false, + checks: [ + { id: 'masking.mode', refs: ['Art. 25'] }, + { id: 'masking.scanned', refs: ['Art. 30'] }, + { id: 'masking.review', refs: ['Art. 5(1)(c)'] }, + { id: 'access.ai', refs: ['Art. 5(1)(c)', 'Art. 25'] }, + { id: 'access.grant_expiry', refs: ['Art. 32'] }, + { id: 'platform.https', refs: ['Art. 32'] }, + { id: 'platform.audit', refs: ['Art. 30'] }, + ], + manual: [ + { + id: 'manual.dpia', + title: 'Impact assessment for high-risk processing', + refs: ['Art. 35'], + }, + ], + rules: [], + }, + { + id: 'soc2', + name: 'SOC 2', + vertical: 'general', + frameworks: ['AICPA SOC 2 Trust Services Criteria (2017)'], + grantMax: 24, + formats: [], + rolesRestricted: false, + adminsAllForbidden: false, + checks: [ + { id: 'masking.mode', refs: ['C1.1'] }, + { id: 'masking.review', refs: ['C1.1'] }, + { id: 'access.ai', refs: ['CC6.1'] }, + { id: 'access.grant_expiry', refs: ['CC6.3'] }, + { id: 'platform.https', refs: ['CC6.7'] }, + { id: 'platform.cors', refs: ['CC6.1'] }, + { id: 'platform.rate_limit', refs: ['CC6.1'] }, + { id: 'platform.audit', refs: ['CC7.2'] }, + ], + manual: [ + { + id: 'manual.access_review', + title: 'Quarterly access review', + refs: ['CC6.3'], + }, + ], + rules: [], + }, + { + id: 'iso27001', + name: 'ISO 27001', + vertical: 'general', + frameworks: ['ISO/IEC 27001:2022'], + grantMax: 24, + formats: [], + rolesRestricted: false, + adminsAllForbidden: false, + checks: [ + { id: 'masking.mode', refs: ['A.8.11'] }, + { id: 'masking.scanned', refs: ['A.5.12'] }, + { id: 'masking.review', refs: ['A.5.12'] }, + { id: 'access.ai', refs: ['A.5.15'] }, + { id: 'access.grant_expiry', refs: ['A.8.2'] }, + { id: 'platform.https', refs: ['A.8.24'] }, + { id: 'platform.audit', refs: ['A.8.15'] }, + ], + manual: [ + { + id: 'manual.classification_policy', + title: 'Information classification policy approved', + refs: ['A.5.12'], + }, + ], + rules: [], + }, +]; + +const TITLES: Record = { + 'masking.mode': 'Masking is on', + 'masking.scanned': 'Every database scanned in the last 30 days', + 'masking.review': 'No columns waiting for review', + 'masking.format': 'Mask formats meet the baseline', + 'access.ai': 'AI and MCP sessions always masked', + 'access.grant_expiry': 'Admin access to real values expires in time', + 'access.admins_all': 'Real values not open to all admins', + 'access.roles_restricted': 'No role sees restricted classes', + 'platform.https': 'Served over HTTPS', + 'platform.session_ttl': 'Sessions expire within 15 minutes', + 'platform.cors': 'No CORS rule allows any origin', + 'platform.rate_limit': 'Rate limits are active', + 'platform.audit': 'Audit logging is on', + 'platform.root_admin': 'A super admin is designated', +}; + +const CLASS_LABEL: Record = { + card_number: 'Card number', + card_security: 'Card security code', + date_of_birth: 'Date of birth', +}; +const FORMAT_LABEL: Record = { + last4: 'Last 4', + redact: 'Redact', + null: 'Null', + year: 'Year only', +}; + +interface Control { + id: string; + title: string; + refs: string[]; + status: 'pass' | 'fail' | 'manual' | 'na'; + detail: string | null; + fix: { kind: 'apply' | 'link' | 'none'; route: string | null }; +} + +function json(route: Route, body: unknown, status = 200) { + return route.fulfill({ + status, + contentType: 'application/json', + body: JSON.stringify(body), + }); +} + +export function installComplianceMock(page: Page, m: MaskingStateView) { + const adopted = new Map(); + /** Baselines whose detection rules were already added. */ + const rulesAdded = new Set(); + + const tooLong = (b: BaselineDef) => + m.access.admins.grants.filter( + g => + g.expires_at === null || + Date.parse(g.expires_at) - Date.now() > b.grantMax * HOUR + 60_000 + ); + const formatMisses = (b: BaselineDef) => + b.formats.filter(f => { + const row = m.masks.find(x => x.class === f.class); + return !!row && !f.allowed.includes(row.format); + }); + const openRoles = () => + m.access.roles.filter(r => r.enabled && r.scope === 'all'); + const review = () => m.columns.filter(c => c.verdict === 'review').length; + + function evaluate(b: BaselineDef, c: Check): Control { + const base = { + id: c.id, + title: c.title ?? TITLES[c.id] ?? c.id, + refs: c.refs, + detail: null as string | null, + }; + const pass = (ok: boolean, fail: Partial = {}): Control => ({ + ...base, + status: ok ? 'pass' : 'fail', + fix: { kind: 'none', route: null }, + ...(ok ? {} : fail), + }); + switch (c.id) { + case 'masking.mode': + return pass(m.mode === 'on', { + detail: + m.mode === 'preview' ? 'Masking is in Preview' : 'Masking is off', + // As the server (Checks::CATALOG): Apply with masking on fixes it. + fix: { kind: 'apply', route: null }, + }); + case 'masking.scanned': + return pass(!!m.scan.finished_at && m.columns.length > 0, { + detail: 'No scan yet', + fix: { kind: 'link', route: '/api-security/data-masking' }, + }); + case 'masking.review': { + const n = review(); + return pass(n === 0, { + detail: `${n} ${n === 1 ? 'column' : 'columns'} to review`, + fix: { + kind: 'link', + route: '/api-security/data-masking?filter=review', + }, + }); + } + case 'masking.format': { + const miss = formatMisses(b); + return { + ...pass(miss.length === 0, { + detail: miss + .map( + f => `${CLASS_LABEL[f.class] ?? f.class} is not masked enough` + ) + .join('; '), + fix: { kind: 'apply', route: null }, + }), + title: b.formats.length === 1 ? b.formats[0].title : base.title, + }; + } + case 'access.ai': + return pass(m.access.ai === 'masked', { + detail: 'AI and MCP sessions see what admins see', + fix: { kind: 'apply', route: null }, + }); + case 'access.grant_expiry': { + const long = tooLong(b); + return { + ...pass(long.length === 0, { + detail: `${long.map(g => g.name).join(', ')}: longer than ${b.grantMax} hours`, + fix: { kind: 'apply', route: null }, + }), + title: `Admin access to real values expires within ${b.grantMax} hours`, + }; + } + case 'access.admins_all': + return pass(m.access.admins.mode !== 'all', { + detail: 'All admins see real values', + fix: { kind: 'link', route: '/api-security/data-masking?tab=access' }, + }); + case 'access.roles_restricted': { + const roles = openRoles(); + return pass(roles.length === 0, { + detail: roles.map(r => r.name).join(', '), + fix: { kind: 'apply', route: null }, + }); + } + case 'platform.https': + case 'platform.audit': + case 'platform.root_admin': + return pass(true); + case 'platform.session_ttl': + return pass(false, { + detail: 'Sessions last 1440 minutes', + fix: { kind: 'none', route: null }, + }); + case 'platform.cors': + return pass(false, { + detail: '1 rule allows any origin', + fix: { kind: 'link', route: '/system-settings/cors' }, + }); + case 'platform.rate_limit': + return c.refs.length + ? pass(false, { + detail: 'No active limits', + fix: { kind: 'link', route: '/api-security/rate-limiting' }, + }) + : { ...base, status: 'na', fix: { kind: 'none', route: null } }; + default: + return { ...base, status: 'na', fix: { kind: 'none', route: null } }; + } + } + + function changes(b: BaselineDef) { + const out: { id: string; text: string }[] = []; + if (m.mode === 'off') { + out.push({ id: 'masking.mode', text: 'Masking mode: Preview' }); + } + for (const f of formatMisses(b)) { + out.push({ + id: `masking.format.${f.class}`, + text: `${CLASS_LABEL[f.class] ?? f.class}: ${FORMAT_LABEL[f.allowed[0]] ?? f.allowed[0]}`, + }); + } + if (m.access.ai !== 'masked') { + out.push({ id: 'access.ai', text: 'AI and MCP sessions: always masked' }); + } + for (const g of tooLong(b)) { + out.push({ + id: `access.grant_expiry.${g.admin_id}`, + text: `${g.name}: real values for at most ${b.grantMax} hours`, + }); + } + if (b.rolesRestricted) { + for (const r of openRoles()) { + out.push({ + id: `access.roles_restricted.${r.role_id}`, + text: `${r.name}: all except restricted`, + }); + } + } + if (b.rules.length && !rulesAdded.has(b.id)) { + out.push({ + id: 'detection.rules', + text: `Add ${b.rules.length} detection ${b.rules.length === 1 ? 'rule' : 'rules'} for ${b.name}`, + }); + } + return out; + } + + function controls(b: BaselineDef): Control[] { + return [ + ...b.checks.map(c => evaluate(b, c)), + ...b.manual.map(x => ({ + ...x, + status: 'manual' as const, + detail: null, + fix: { kind: 'none' as const, route: null }, + })), + ]; + } + + function summary(b: BaselineDef) { + const cs = controls(b); + return { + id: b.id, + name: b.name, + vertical: b.vertical, + frameworks: b.frameworks, + adopted: adopted.has(b.id), + score: { + pass: cs.filter(c => c.status === 'pass').length, + fail: cs.filter(c => c.status === 'fail').length, + manual: cs.filter(c => c.status === 'manual').length, + total: cs.length, + }, + }; + } + + function detail(b: BaselineDef) { + const a = adopted.get(b.id); + return { + id: b.id, + name: b.name, + vertical: b.vertical, + frameworks: b.frameworks, + adopted: !!a, + adopted_at: a?.at ?? null, + applied_at: a?.applied ?? null, + controls: controls(b), + changes: changes(b), + }; + } + + function apply(b: BaselineDef, turnOn: boolean) { + const now = new Date().toISOString(); + if (m.mode === 'off') m.mode = 'preview'; + for (const f of formatMisses(b)) { + const row = m.masks.find(x => x.class === f.class)!; + row.format = f.allowed[0]; + } + m.access.ai = 'masked'; + for (const g of tooLong(b)) { + g.expires_at = new Date(Date.now() + b.grantMax * HOUR).toISOString(); + } + if (b.rolesRestricted) { + openRoles().forEach(r => (r.scope = 'except_restricted')); + } + if (b.rules.length && !rulesAdded.has(b.id)) { + for (const r of b.rules) m.rules.push({ id: m.nextRule++, ...r }); + rulesAdded.add(b.id); + } + if (turnOn) m.mode = 'on'; + adopted.set(b.id, { at: adopted.get(b.id)?.at ?? now, applied: now }); + m.ev(0, ME, 'baseline', `Applied the ${b.name} baseline`); + if (turnOn) m.ev(0, ME, 'mode', 'Masking set to On'); + } + + function report(b: BaselineDef, format: string) { + const d = detail(b); + if (format === 'csv') { + const esc = (v: string) => `"${v.replace(/"/g, '""')}"`; + const rows = d.controls.map(c => + [c.id, c.title, c.status, c.refs.join('; '), c.detail ?? ''] + .map(esc) + .join(',') + ); + return ['control,title,status,refs,detail', ...rows].join('\n') + '\n'; + } + const inventory: Record = {}; + for (const c of m.columns) + inventory[c.verdict] = (inventory[c.verdict] ?? 0) + 1; + return JSON.stringify( + { + instance: 'mock', + generated_at: new Date().toISOString(), + baseline: { id: b.id, name: b.name, frameworks: b.frameworks }, + controls: d.controls.map(c => ({ + id: c.id, + title: c.title, + status: c.status, + refs: c.refs, + detail: c.detail, + })), + inventory, + access: { + admins: m.access.admins.mode, + ai: m.access.ai, + grants: m.access.admins.grants.length, + }, + }, + null, + 2 + ); + } + + return page.route('**/api/v2/system/security_baseline**', async route => { + const req = route.request(); + const url = new URL(req.url()); + const method = req.method(); + const sub = url.pathname.replace(/^.*\/system\/security_baseline/, ''); + const body = (() => { + try { + return req.postDataJSON() ?? {}; + } catch { + return {}; + } + })(); + if (sub === '' || sub === '/') { + return json(route, { resource: BASELINES.map(summary) }); + } + const match = sub.match(/^\/([a-z0-9_]+)(\/apply|\/report)?$/); + const b = match && BASELINES.find(x => x.id === match[1]); + if (!match || !b) { + return json(route, { error: { code: 404, message: 'Not found' } }, 404); + } + const action = match[2] ?? ''; + if (m.forbidWrites && method !== 'GET') { + return json( + route, + { + error: { + code: 403, + // The server names what it refused (security_baseline). + message: m.forbidWrites.replace( + 'Data Masking settings', + 'Compliance baselines' + ), + status_code: 403, + }, + }, + 403 + ); + } + if (action === '/apply' && method === 'POST') { + // As the server: no masking On before the first scan; Apply on a + // never-scanned instance starts that scan (scan_started). + const neverScanned = !m.scan.finished_at; + if (body.mode === 'on' && neverScanned) { + return json( + route, + { + error: { + code: 409, + message: 'Scan your databases before turning masking on.', + }, + }, + 409 + ); + } + apply(b, body.mode === 'on'); + const scanStarted = neverScanned && m.scan.status !== 'running'; + if (scanStarted) { + m.scan.status = 'running'; + m.scan.progress = 0; + } + return json(route, { ...detail(b), scan_started: scanStarted }); + } + if (action === '/report' && method === 'GET') { + const format = url.searchParams.get('format') ?? 'json'; + return route.fulfill({ + status: 200, + contentType: format === 'csv' ? 'text/csv' : 'application/json', + body: report(b, format), + }); + } + if (action === '' && method === 'DELETE') { + adopted.delete(b.id); + m.ev(0, ME, 'baseline', `Stopped tracking the ${b.name} baseline`); + return json(route, detail(b)); + } + // GET ?mode=on: what Apply with masking turned on would change. + if (url.searchParams.get('mode') === 'on' && m.mode !== 'on') { + const d = detail(b); + return json(route, { + ...d, + changes: [ + { + id: 'mode', + text: + m.mode === 'preview' + ? 'Turn masking on (now Preview).' + : 'Turn masking on.', + }, + ...d.changes.filter(c => c.id !== 'masking.mode'), + ], + }); + } + return json(route, detail(b)); + }); +} diff --git a/e2e/fixtures/data-masking-mock.ts b/e2e/fixtures/data-masking-mock.ts new file mode 100644 index 000000000..c7410c2bd --- /dev/null +++ b/e2e/fixtures/data-masking-mock.ts @@ -0,0 +1,1268 @@ +/** + * Playwright route mock for the Data Masking backend + * (/api/v2/system/data_masking/**) and, through compliance-mock.ts, the + * Compliance baselines (/api/v2/system/security_baseline/**). It follows + * the backend contract (df-masking, section 8) so screenshots and UI flows + * can run without a GOLD backend; the live spec covers the real one. + * + * - Seeds from fixtures/data-masking/fixture.json: real classifier output for + * three live databases (crm, sports, store), with humanized evidence and + * masked renderings of real first-row values. No raw values. + * - Keeps in-memory state per page, so mode changes, decisions, grants, + * formats and rules persist for the session, and every write lands in + * Activity. + * - Speaks the wire format (snake_case); the app's case interceptor + * camelCases responses like any other system API. + * - Reports the chosen license and, when the package counts as installed + * (default: GOLD only), adds the `data_masking` / `security_baseline` + * system resources so the pages open instead of the paywall. + * + * Usage: + * await installDataMaskingMock(page); // GOLD, seeded + * await installDataMaskingMock(page, { license: 'OPEN SOURCE' }); // paywall + * await installDataMaskingMock(page, { installed: false }); // GOLD, paywall + * await installDataMaskingMock(page, { scenario: 'unscanned' }); + * await installDataMaskingMock(page, { scanUnknown: true }); // history unreadable + */ +import { Page, Route } from '@playwright/test'; +import fixture from './data-masking/fixture.json'; +import { installComplianceMock } from './compliance-mock'; +import i18n from '../../src/assets/i18n/dataMasking/en.json'; +import { + CLASSES, + MaskFormat, + classDef, +} from '../../src/app/adf-data-masking/data-masking.catalog'; +import { + applyMask, + sampleMask, +} from '../../src/app/adf-data-masking/mask-format'; + +export interface DataMaskingMockOptions { + license?: 'GOLD' | 'SILVER' | 'OPEN SOURCE'; + /** + * Whether the df-masking package's system resources exist. Default: on + * GOLD only. Without them every license shows the standard paywall. + */ + installed?: boolean; + /** + * default: scanned, 24 columns to review. + * unscanned: nothing scanned yet; Scan now fills it from the fixture. + * reviewed: every review column already decided. + * nogrant: the signed-in admin cannot give real-value access. + * nodb: a brand-new instance with no SQL database to scan. + * emptydb: a default install: only the built-in `db` (SQLite, no tables). + */ + scenario?: + | 'default' + | 'unscanned' + | 'reviewed' + | 'nogrant' + | 'nodb' + | 'emptydb'; + /** Default: preview when scanned, off when unscanned (as the server). */ + mode?: 'off' | 'preview' | 'on'; + /** List a non-SQL database Data Masking cannot cover (default true). */ + uncovered?: boolean; + /** Add a covered SQL service that was never scanned (default false). */ + pending?: boolean; + /** + * Add a covered SQL service whose database cannot be reached: every scan + * that includes it fails for it (default false). + */ + unreachable?: boolean; + /** + * Refuse every Data Masking and Compliance write with 403 and this + * message, as the server does for a caller it does not let change + * settings. Reads still work. + */ + forbidWrites?: string; + /** + * Add a computed field the scanner could not check (an expression over a + * masked column): stored as review with no class, so it is redacted. + */ + computed?: boolean; + /** + * The server cannot read its scan history (a database error): the + * overview's scan says `unknown: true` with no finished_at, last_status + * or failures, and every /table entry says `scan_unknown: true` with no + * scan time. Classified columns still list (default false). + */ + scanUnknown?: boolean; +} + +/** Non-SQL database services: humans unaffected, AI blocked when on. */ +const UNCOVERED = [ + { service: 'events', label: 'Event store', type: 'mongodb' }, +]; +/** A covered SQL service added after the last scan. */ +const PENDING = { service: 'hr', label: 'HR (SQL Server)', tables: [] }; +/** A covered SQL service whose database refuses the connection. */ +const UNREACHABLE = { service: 'orders', label: 'Orders (MySQL)', tables: [] }; +const UNREACHABLE_REASON = 'Could not connect to the database.'; +/** A default install's built-in SQLite service: covered, no tables. */ +const BUILTIN_DB = { service: 'db', label: 'Local SQL Database', tables: [] }; + +type Verdict = 'masked' | 'review' | 'clear'; + +interface FixColumn { + id: string; + service: string; + table: string; + column: string; + type: string; + verdict: Verdict; + class: string | null; + tier: string | null; + confidence: number; + total: number; + lines: [number, string, string][]; + shapes?: [string, number][]; + examples?: Record; +} + +interface Col extends FixColumn { + source: 'detected' | 'admin'; + decided_by: string | null; + decided_at: string | null; + extra: { points: number; kind: string; text: string }[]; +} + +const ME = { admin_id: 1, name: 'Kevin Admin', email: 'admin@dmui.local' }; +const ADMINS = [ + ME, + { admin_id: 2, name: 'Priya Shah', email: 'priya@example.com' }, + { admin_id: 3, name: 'Marco Rossi', email: 'marco@example.com' }, + { admin_id: 4, name: 'Dana Okafor', email: 'dana@example.com' }, + { admin_id: 5, name: 'Lee Chen', email: 'lee@example.com' }, +]; +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +const DURATIONS: Record = { + '1h': HOUR, + '8h': 8 * HOUR, + '24h': DAY, + '7d': 7 * DAY, + forever: null, +}; +const CLASS_LABEL = (id: string | null) => + id ? ((i18n.classes as Record)[id] ?? id) : i18n.notSensitive; +const FORMAT_LABEL = (f: string) => + ((i18n.formats as Record)[f] ?? f).toLowerCase(); + +function json(route: Route, body: unknown, status = 200) { + return route.fulfill({ + status, + contentType: 'application/json', + body: JSON.stringify(body), + }); +} + +function buildState(opts: DataMaskingMockOptions) { + const now = Date.now(); + const iso = (msAgo: number) => new Date(now - msAgo).toISOString(); + const fx0 = fixture as unknown as { + meta: { ruleset: string; version: string }; + services: { service: string; label: string; tables: string[] }[]; + builtin: { class: string; detectors: string[] }[]; + columns: FixColumn[]; + }; + const empty = opts.scenario === 'nodb' || opts.scenario === 'emptydb'; + const fx = empty ? { ...fx0, services: [], columns: [] } : fx0; + + const seedColumns = (): Col[] => + fx.columns.map(c => ({ + ...c, + source: 'detected', + decided_by: null, + decided_at: null, + extra: [], + })); + + const scanned = opts.scenario !== 'unscanned' && !empty; + const state = { + mode: opts.mode ?? (scanned ? 'preview' : 'off'), + scan: { + status: 'idle' as 'idle' | 'running', + progress: 0, + finished_at: scanned ? iso(2 * HOUR) : (null as string | null), + last_status: (scanned ? 'done' : null) as + | 'done' + | 'partial' + | 'failed' + | null, + failed: [] as { + service_id: number; + service: string; + label: string; + reason: string; + }[], + /** Services the running scan covers (null = all). */ + only: null as string | null, + }, + /** Opaque access version: bumped by every access write. */ + accessVersion: 1, + /** When each covered service was last scanned (null = never). */ + scannedAt: new Map(), + /** Why each service's last scan failed (absent = it did not). */ + scanError: new Map(), + builtinDb: opts.scenario === 'emptydb', + unreachable: !!opts.unreachable, + columns: scanned ? seedColumns() : ([] as Col[]), + masks: CLASSES.map(c => ({ + class: c.id, + tier: c.tier, + format: c.format as MaskFormat, + formats_allowed: c.formats, + })), + access: { + admins: { + mode: 'selected' as 'none' | 'selected' | 'all', + grants: [ + { ...ME, expires_at: new Date(now + 8 * HOUR).toISOString() }, + ] as { + admin_id: number; + name: string; + email: string; + expires_at: string | null; + }[], + }, + ai: 'masked', + roles: [ + { + role_id: 1, + name: 'Support agents', + enabled: true, + scope: 'except_restricted', + }, + { role_id: 2, name: 'Billing service', enabled: true, scope: 'all' }, + { + role_id: 3, + name: 'Mobile app', + enabled: false, + scope: 'except_restricted', + }, + { + role_id: 4, + name: 'Analytics', + enabled: false, + scope: 'except_restricted', + }, + ], + }, + rules: [] as { + id: number; + match: 'name' | 'value'; + pattern: string; + class: string; + }[], + nextRule: 2, + activity: [] as { + id: number; + at: string; + actor: string; + kind: string; + text: string; + target?: string | null; + }[], + nextEvent: 100, + canGrant: opts.scenario !== 'nogrant', + forbidWrites: opts.forbidWrites ?? null, + uncovered: opts.uncovered === false ? [] : UNCOVERED, + pending: !!opts.pending, + scanUnknown: !!opts.scanUnknown, + fx, + seedColumns, + }; + for (const svc of fx.services) { + state.scannedAt.set(svc.service, scanned ? state.scan.finished_at : null); + } + + const decide = (id: string, cls: string | null, by: string, at: string) => { + const c = state.columns.find(x => x.id === id); + if (!c) return; + c.verdict = cls ? 'masked' : 'clear'; + c.class = cls; + c.tier = classDef(cls)?.tier ?? null; + c.source = 'admin'; + c.decided_by = by; + c.decided_at = at; + }; + + if (scanned && opts.computed) { + // DerivedFields: an expression that can't be checked is review with no + // class (the server redacts it); evidence names it, never values. + const base = state.columns.find( + c => c.service === 'crm' && c.verdict === 'masked' + ); + if (base) { + state.columns.push({ + id: `${base.service}.${base.table}.contact_line`, + service: base.service, + table: base.table, + column: 'contact_line', + type: 'virtual', + verdict: 'review', + class: null, + tier: null, + confidence: 40, + total: 40, + lines: [ + [40, 'context', 'Built from an expression that could not be checked'], + ], + source: 'detected', + decided_by: null, + decided_at: null, + extra: [], + }); + } + } + + if (scanned) { + // A few earlier admin decisions, so sources and tooltips have history. + decide( + 'crm.CustomerContacts.AccountName', + 'username', + 'Priya Shah', + iso(2 * DAY) + ); + decide('crm.CustomerContacts.LeadSource', null, 'Priya Shah', iso(2 * DAY)); + decide( + 'crm.hr_staff.salary', + 'financial_personal', + 'Kevin Admin', + iso(5 * DAY) + ); + if (opts.scenario === 'reviewed') { + state.columns + .filter(c => c.verdict === 'review') + .forEach(c => decide(c.id, c.class, 'Kevin Admin', iso(HOUR))); + } + state.rules.push({ + id: 1, + match: 'name', + pattern: '(^|_)(ssn|nino|bsn)($|_)', + class: 'national_id', + }); + } + + const ev = ( + msAgo: number, + actor: string, + kind: string, + text: string, + target?: string + ) => + state.activity.push({ + id: state.nextEvent++, + at: iso(msAgo), + actor, + kind, + text, + target: target ?? null, + }); + + if (scanned) { + const total = state.columns.length; + ev( + 12 * 60_000, + 'Kevin Admin', + 'access', + 'Kevin Admin was given real values for 8 hours' + ); + ev( + 40 * 60_000, + 'Claude Desktop (AI session)', + 'blocked', + 'Blocked an attempt to change access' + ); + ev( + 2 * HOUR, + 'System', + 'scan', + `Scan finished: ${total} columns in ${fx.services.length} databases` + ); + ev(26 * HOUR, 'Priya Shah', 'mode', 'Masking set to Preview'); + ev( + 27 * HOUR, + 'Priya Shah', + 'rule', + 'Added a column name rule for National ID', + '(^|_)(ssn|nino|bsn)($|_)' + ); + ev( + 2 * DAY, + 'Priya Shah', + 'decision', + 'Kept masked as Username', + 'crm.CustomerContacts.AccountName' + ); + ev( + 2 * DAY + 60_000, + 'Priya Shah', + 'decision', + 'Marked not sensitive', + 'crm.CustomerContacts.LeadSource' + ); + ev( + 3 * DAY, + 'Dana Okafor', + 'blocked', + 'A filter on a masked column ran on masked values', + 'crm.CustomerContacts.EMail' + ); + ev( + 5 * DAY, + 'Kevin Admin', + 'decision', + 'Kept masked as Personal financial', + 'crm.hr_staff.salary' + ); + ev( + 5 * DAY + HOUR, + 'System', + 'scan', + `Scan finished: ${total} columns in ${fx.services.length} databases` + ); + } + + return Object.assign(state, { ev, decide, iso }); +} + +type State = ReturnType; + +function formatOf(state: State, cls: string | null): MaskFormat | null { + return (state.masks.find(m => m.class === cls)?.format as MaskFormat) ?? null; +} + +function ruleMatches(state: State, match: 'name' | 'value', pattern: string) { + let re: RegExp; + try { + re = new RegExp(pattern, 'i'); + } catch { + return []; + } + return state.columns.filter(c => { + if (match === 'name') return re.test(c.column); + // No raw values in the mock: test value shapes and the class sample. + const probes = [ + ...(c.shapes ?? []).map(s => s[0]), + ...(c.verdict !== 'clear' && c.class + ? [classDef(c.class)?.sample ?? ''] + : []), + ]; + return probes.some(p => re.test(p)); + }); +} + +function applyRule( + state: State, + rule: { match: 'name' | 'value'; pattern: string; class: string } +) { + // Rules only ever add masking, and never override an admin decision. + for (const c of ruleMatches(state, rule.match, rule.pattern)) { + if (c.source === 'admin' || c.verdict === 'masked') continue; + c.verdict = 'masked'; + c.class = rule.class; + c.tier = classDef(rule.class)?.tier ?? null; + // The scanner's evidence line (Rules::describe). + const kind = rule.match === 'value' ? 'value rule' : 'rule'; + c.extra = [ + { points: 70, kind: 'context', text: `Custom ${kind} ${rule.pattern}` }, + ]; + c.total = Math.max(c.total, 70); + } +} + +function wireColumn(state: State, c: Col) { + const format = c.verdict === 'clear' ? null : formatOf(state, c.class); + let example: string | null = null; + if (format && c.class) { + example = + c.examples && format in c.examples + ? (c.examples[format] as string | null) + : sampleMask(c.class, format).after; + } + return { + id: c.id, + service: c.service, + table: c.table, + column: c.column, + type: c.type, + verdict: c.verdict, + class: c.class, + tier: c.tier, + source: c.source, + decided_by: c.decided_by, + decided_at: c.decided_at, + confidence: c.confidence, + example, + format, + evidence: { + total: c.total, + threshold_mask: 70, + threshold_review: 40, + lines: [ + ...c.lines.map(([points, kind, text]) => ({ points, kind, text })), + ...c.extra, + ], + shapes: c.shapes ?? [], + }, + }; +} + +function counts(state: State) { + const out = { masked: 0, review: 0, clear: 0 }; + state.columns.forEach(c => out[c.verdict]++); + return out; +} + +function overview(state: State) { + const mine = state.access.admins.grants.find(g => g.admin_id === ME.admin_id); + const live = + state.access.admins.mode === 'all' || + (state.access.admins.mode === 'selected' && + !!mine && + (mine.expires_at === null || Date.parse(mine.expires_at) > Date.now())); + return { + mode: state.mode, + scan: { + status: state.scan.status, + progress: state.scan.status === 'running' ? state.scan.progress : null, + columns: state.columns.length, + // An unreadable scan history: nothing is known about past scans. + ...(state.scanUnknown + ? { finished_at: null, last_status: null, failed: [], unknown: true } + : { + finished_at: state.scan.finished_at, + last_status: state.scan.last_status, + failed: state.scan.failed, + }), + }, + counts: counts(state), + you: { + can_grant: state.canGrant, + sees_real_values: live, + grant_expires_at: live ? (mine?.expires_at ?? null) : null, + }, + coverage: { + covered: [ + ...state.fx.services.map(s => ({ + service: s.service, + label: s.label, + type: 'pgsql', + })), + ...(state.pending + ? [{ service: PENDING.service, label: PENDING.label, type: 'sqlsrv' }] + : []), + ...(state.unreachable + ? [ + { + service: UNREACHABLE.service, + label: UNREACHABLE.label, + type: 'mysql', + }, + ] + : []), + ...(state.builtinDb + ? [ + { + service: BUILTIN_DB.service, + label: BUILTIN_DB.label, + type: 'sqlite', + }, + ] + : []), + ], + uncovered: state.uncovered, + }, + }; +} + +function tables(state: State) { + // Like the server: every covered service; one never scanned has no + // tables and no scan time, one the last scan could not read says why. + const services = [ + ...state.fx.services.map(s => { + const at = state.scannedAt.get(s.service) ?? null; + return { ...s, scanned_at: at, tables: at ? s.tables : [] }; + }), + ...(state.pending ? [{ ...PENDING, scanned_at: null }] : []), + ...(state.unreachable + ? [ + { + ...UNREACHABLE, + scanned_at: state.scannedAt.get(UNREACHABLE.service) ?? null, + }, + ] + : []), + ...(state.builtinDb + ? [ + { + ...BUILTIN_DB, + scanned_at: state.scannedAt.get(BUILTIN_DB.service) ?? null, + }, + ] + : []), + ]; + return { + resource: services.map(s => ({ + service: s.service, + label: s.label, + // An unreadable scan history: no scan time or error, only the flag. + scanned_at: state.scanUnknown ? null : s.scanned_at, + scan_error: state.scanUnknown + ? null + : (state.scanError.get(s.service) ?? null), + ...(state.scanUnknown ? { scan_unknown: true } : {}), + tables: (s.tables as string[]).map(name => { + const cols = state.columns.filter( + c => c.service === s.service && c.table === name + ); + const columns = { masked: 0, review: 0, clear: 0 }; + cols.forEach(c => columns[c.verdict]++); + return { + name, + columns, + strip: cols.map(c => ({ + column: c.column, + mark: + c.verdict === 'review' + ? 'review' + : c.verdict === 'clear' + ? 'clear' + : c.tier, + tier: c.tier, + })), + }; + }), + })), + }; +} + +const ADMIN_RANK: Record = { none: 0, selected: 1, all: 2 }; + +/** The server's loosening rule (AccessHandler): more admins, a new or + * longer grant, AI same as admins, a role enabled or widened to all. */ +interface AccessBody { + admins?: { + mode?: string; + grants?: { admin_id: number; duration?: string }[]; + }; + ai?: string; + roles?: { role_id: number; enabled: boolean; scope: string }[]; +} + +function loosens(state: State, body: AccessBody): boolean { + const now = Date.now(); + const a = body.admins ?? {}; + if (a.mode && ADMIN_RANK[a.mode] > ADMIN_RANK[state.access.admins.mode]) { + return true; + } + if (body.ai === 'same_as_admins' && state.access.ai !== 'same_as_admins') { + return true; + } + for (const g of Array.isArray(a.grants) ? a.grants : []) { + const old = state.access.admins.grants.find( + p => + p.admin_id === g.admin_id && + (p.expires_at === null || Date.parse(p.expires_at) > now) + ); + if (!old) return true; + if (g.duration && old.expires_at !== null) { + const ms = DURATIONS[g.duration]; + if (ms === null || now + (ms ?? 0) > Date.parse(old.expires_at)) { + return true; + } + } + } + for (const r of Array.isArray(body.roles) ? body.roles : []) { + const old = state.access.roles.find(x => x.role_id === r.role_id); + if (r.enabled && !old?.enabled) return true; + if (r.enabled && old?.enabled && r.scope === 'all' && old.scope !== 'all') { + return true; + } + } + return false; +} + +function access(state: State) { + return { + admins: { + mode: state.access.admins.mode, + grants: state.access.admins.grants, + available: ADMINS.map(a => ({ ...a, you: a.admin_id === ME.admin_id })), + }, + ai: state.access.ai, + roles: state.access.roles, + version: `v${state.accessVersion}`, + }; +} + +/** A scan reached 100%: classify, record what could not be read. */ +function finishScan(state: State) { + const now = new Date().toISOString(); + const only = state.scan.only; + const covers = (svc: string) => only === null || only === svc; + state.scan.status = 'idle'; + state.scan.only = null; + const failed: typeof state.scan.failed = []; + if (state.unreachable && covers(UNREACHABLE.service)) { + failed.push({ + service_id: 90, + service: UNREACHABLE.service, + label: UNREACHABLE.label, + reason: UNREACHABLE_REASON, + }); + state.scanError.set(UNREACHABLE.service, UNREACHABLE_REASON); + } + const done = [ + ...state.fx.services.map(s => s.service), + ...(state.builtinDb ? [BUILTIN_DB.service] : []), + ].filter(covers); + const first = !state.columns.length && !!state.fx.services.length; + if ( + first && + done.some(svc => state.fx.services.some(f => f.service === svc)) + ) { + state.columns = state.seedColumns(); + state.rules.forEach(r => applyRule(state, r)); + } + done.forEach(svc => { + state.scannedAt.set(svc, now); + state.scanError.delete(svc); + }); + state.scan.failed = failed; + state.scan.last_status = !failed.length + ? 'done' + : done.length + ? 'partial' + : 'failed'; + // A failed scan does not move finished_at (the last done scan's time). + if (state.scan.last_status !== 'failed') state.scan.finished_at = now; + const firstEver = done.length > 0 && first; + if (failed.length) { + state.ev( + 0, + 'System', + 'scan', + `Scan failed: could not read ${failed.map(f => f.service).join(', ')}` + ); + } + if (done.length) { + state.ev( + 0, + 'System', + 'scan', + `Scan finished: ${state.columns.length} columns in ${done.length} databases` + ); + } + // The server moves Off to Preview after the first scan ever. + if ((firstEver || (state.builtinDb && done.length)) && state.mode === 'off') { + state.mode = 'preview'; + state.ev( + 0, + 'System', + 'mode', + 'Masking set to Preview after the first scan' + ); + } +} + +async function handle(route: Route, state: State) { + const req = route.request(); + const url = new URL(req.url()); + const method = req.method(); + const sub = url.pathname.replace(/^.*\/system\/data_masking/, '') || '/'; + const body = (() => { + try { + return req.postDataJSON() ?? {}; + } catch { + return {}; + } + })(); + + // A caller the server does not let change settings: every write is + // refused (the rule preview is a read, even though it is a POST). + if (state.forbidWrites && method !== 'GET' && sub !== '/rule/test') { + state.ev(0, ME.name, 'blocked', 'Blocked an attempt to change settings'); + return json( + route, + { error: { code: 403, message: state.forbidWrites, status_code: 403 } }, + 403 + ); + } + + // ------------------------------------------------------------ overview + if (sub === '/' || sub === '') { + if (method === 'PATCH' && body.mode) { + const rank: Record = { off: 0, preview: 1, on: 2 }; + const neverScanned = ![...state.scannedAt.values()].some(Boolean); + // Like the server: nothing to mask before the first scan. + if (body.mode === 'on' && neverScanned) { + return json( + route, + { + error: { + code: 409, + message: 'Scan your databases before turning masking on.', + }, + }, + 409 + ); + } + // Lowering from On is the super admin's call. + if ( + state.mode === 'on' && + rank[body.mode] < rank['on'] && + !state.canGrant + ) { + state.ev(0, ME.name, 'blocked', 'Blocked an attempt to lower the mode'); + return json( + route, + { + error: { + code: 403, + message: 'Only the super admin can lower the masking mode.', + }, + }, + 403 + ); + } + state.mode = body.mode; + const label = { off: 'Off', preview: 'Preview', on: 'On' }[ + body.mode as string + ]; + state.ev(0, ME.name, 'mode', `Masking set to ${label}`); + } + if (method === 'GET' && state.scan.status === 'running') { + state.scan.progress = Math.min(100, state.scan.progress + 25); + if (state.scan.progress >= 100) finishScan(state); + } + return json(route, overview(state)); + } + + if (sub === '/scan' && method === 'POST') { + // A scan already running: the overview, unchanged (idempotent). + if (state.scan.status !== 'running') { + state.scan.status = 'running'; + state.scan.progress = 0; + state.scan.only = (body.service as string | undefined) ?? null; + } + return json(route, overview(state)); + } + + if (sub === '/table') return json(route, tables(state)); + + // ------------------------------------------------------------- columns + if (sub === '/column') { + const svc = url.searchParams.get('service'); + return json(route, { + resource: state.columns + .filter(c => !svc || c.service === svc) + .map(c => wireColumn(state, c)), + }); + } + const colMatch = sub.match(/^\/column(?:\/(.+))?$/); + if (colMatch && method === 'PATCH') { + // The id travels in the body (any column name); the path form is kept. + const id = + (body.id as string | undefined) ?? decodeURIComponent(colMatch[1] ?? ''); + const c = state.columns.find(x => x.id === id); + if (!c) return json(route, { error: { message: 'Column not found' } }, 404); + const wasReview = c.verdict === 'review'; + const cls = (body.class ?? null) as string | null; + const sameClass = cls === c.class; + state.decide(id, cls, ME.name, new Date().toISOString()); + const label = classDef(cls) ? CLASS_LABEL(cls) : ''; + state.ev( + 0, + ME.name, + 'decision', + !cls + ? 'Marked not sensitive' + : wasReview && sameClass + ? `Kept masked as ${label}` + : `Set class to ${label}`, + id + ); + return json(route, wireColumn(state, c)); + } + + // -------------------------------------------------------------- access + if (sub === '/access') { + if (method === 'PATCH') { + // Optimistic concurrency, like the server: a stale version is refused. + if (body.version && body.version !== `v${state.accessVersion}`) { + return json( + route, + { + error: { + code: 409, + message: + 'Access settings changed since you loaded them. Reload and try again.', + }, + }, + 409 + ); + } + // Like the server: anyone may tighten; loosening needs the super + // admin, and a PATCH with any loosening change fails as a whole. + if (!state.canGrant && loosens(state, body)) { + state.ev(0, ME.name, 'blocked', 'Blocked an attempt to change access'); + return json( + route, + { + error: { + code: 403, + message: 'Only the super admin can give access.', + }, + }, + 403 + ); + } + state.accessVersion++; + const a = body.admins ?? {}; + if (a.mode) state.access.admins.mode = a.mode; + if (Array.isArray(a.grants)) { + const prev = state.access.admins.grants; + const now = Date.now(); + const live = (p: { expires_at: string | null }) => + p.expires_at === null || Date.parse(p.expires_at) > now; + state.access.admins.grants = a.grants + // A grant sent without a duration only keeps a live grant; it + // never brings back one that ended or was revoked. + .filter( + (g: { admin_id: number; duration?: string }) => + !!g.duration || + prev.some(p => p.admin_id === g.admin_id && live(p)) + ) + .map((g: { admin_id: number; duration?: string }) => { + const who = ADMINS.find(x => x.admin_id === g.admin_id) ?? ME; + const old = prev.find(p => p.admin_id === g.admin_id); + const ms = g.duration ? DURATIONS[g.duration] : undefined; + const expires_at = + g.duration === undefined && old + ? old.expires_at + : ms === null + ? null + : new Date(Date.now() + (ms ?? 8 * HOUR)).toISOString(); + if (g.duration) { + const span = + g.duration === 'forever' + ? 'until turned off' + : `for ${g.duration + .replace('h', ' hours') + .replace('d', ' days') + .replace(/^1 hours/, '1 hour')}`; + state.ev( + 0, + ME.name, + 'access', + `${who.name} was given real values ${span}` + ); + } + return { ...who, expires_at }; + }); + prev + .filter( + p => + !state.access.admins.grants.some(g => g.admin_id === p.admin_id) + ) + .forEach(p => + state.ev(0, ME.name, 'access', `Real values removed for ${p.name}`) + ); + } + if (body.ai) { + state.access.ai = body.ai; + state.ev( + 0, + ME.name, + 'access', + body.ai === 'masked' + ? 'AI and MCP sessions set to always masked' + : 'AI and MCP sessions now match admins' + ); + } + if (Array.isArray(body.roles)) { + for (const r of body.roles) { + const role = state.access.roles.find(x => x.role_id === r.role_id); + if (!role) continue; + if (role.enabled !== r.enabled) { + state.ev( + 0, + ME.name, + 'access', + `${role.name} ${r.enabled ? 'now sees real values' : 'is masked again'}` + ); + } + Object.assign(role, { enabled: r.enabled, scope: r.scope }); + } + } + if (a.mode) + state.ev( + 0, + ME.name, + 'access', + `Admin access set to ${a.mode === 'none' ? 'no one' : a.mode}` + ); + } + return json(route, access(state)); + } + + // --------------------------------------------------------------- masks + if (sub === '/mask') { + if (method === 'PATCH' && Array.isArray(body.resource)) { + for (const m of body.resource) { + const row = state.masks.find(x => x.class === m.class); + if (row && row.formats_allowed.includes(m.format)) { + row.format = m.format; + state.ev( + 0, + ME.name, + 'decision', + `${CLASS_LABEL(m.class)} now masked as ${FORMAT_LABEL(m.format)}` + ); + } + } + } + return json(route, { resource: state.masks }); + } + + // --------------------------------------------------------------- rules + if (sub === '/rule/builtin') { + return json(route, { + ruleset: state.fx.meta.ruleset, + version: state.fx.meta.version, + classes: state.fx.builtin, + }); + } + if (sub === '/rule/test' && method === 'POST') { + return json(route, { + matches: ruleMatches(state, body.match, body.pattern).map(c => ({ + id: c.id, + })), + partial: false, + }); + } + if (sub === '/rule') { + if (method === 'POST') { + const rule = { + id: state.nextRule++, + match: body.match, + pattern: body.pattern, + class: body.class, + }; + state.rules.push(rule); + applyRule(state, rule); + state.ev( + 0, + ME.name, + 'rule', + `Added a ${rule.match === 'name' ? 'column name' : 'value'} rule for ${CLASS_LABEL(rule.class)}`, + rule.pattern + ); + return json(route, { + ...rule, + matches: ruleMatches(state, rule.match, rule.pattern).length, + }); + } + return json(route, { + resource: state.rules.map(r => ({ + ...r, + matches: ruleMatches(state, r.match, r.pattern).length, + })), + }); + } + const ruleMatch = sub.match(/^\/rule\/(\d+)$/); + if (ruleMatch && method === 'DELETE') { + const id = Number(ruleMatch[1]); + const r = state.rules.find(x => x.id === id); + state.rules = state.rules.filter(x => x.id !== id); + if (r) state.ev(0, ME.name, 'rule', 'Deleted a rule', r.pattern); + return json(route, { id }); + } + + // ------------------------------------------------------------ activity + if (sub === '/activity') { + // Server-side filter and paging: kind=a,b&limit=n&before=. + const kinds = (url.searchParams.get('kind') ?? '') + .split(',') + .filter(Boolean); + const limit = Math.min(1000, Number(url.searchParams.get('limit')) || 200); + const before = Number(url.searchParams.get('before')) || 0; + const sorted = [...state.activity] + .filter(e => !kinds.length || kinds.includes(e.kind)) + .sort((a, b) => Date.parse(b.at) - Date.parse(a.at) || b.id - a.id); + // Older than `before`: everything after that row in newest-first order. + const at = before ? sorted.findIndex(e => e.id === before) : -1; + const all = !before + ? sorted + : at >= 0 + ? sorted.slice(at + 1) + : sorted.filter(e => e.id < before); + return json(route, { + resource: all.slice(0, limit), + has_more: all.length > limit, + }); + } + + return json( + route, + { error: { message: `No mock for ${method} ${sub}` } }, + 404 + ); +} + +export async function installDataMaskingMock( + page: Page, + opts: DataMaskingMockOptions = {} +) { + const license = opts.license ?? 'GOLD'; + const installed = opts.installed ?? license === 'GOLD'; + const state = buildState(opts); + + // Never call out to the license / paywall tracking service. + await page.route('https://updates.dreamfactory.com/**', r => + json(r, { disableUi: 'false', msg: 'Mock license' }) + ); + + await page.route('**/api/v2/system/environment*', async route => { + const res = await route.fetch(); + const env = await res.json(); + env.platform = { ...(env.platform ?? {}), license }; + // The license guard checks a commercial key against the (mocked) + // updates service, so give the mock license one. + if (license !== 'OPEN SOURCE') env.platform.license_key = 'dm-mock-license'; + return route.fulfill({ response: res, json: env }); + }); + + // The resource list the paywall checks (GET /api/v2/system). + await page.route(/\/api\/v2\/system(\?[^/]*)?$/, async route => { + const res = await route.fetch(); + const sys = await res.json(); + const gold = ['data_masking', 'security_baseline']; + if (installed && Array.isArray(sys.resource)) { + for (const name of gold) { + const has = sys.resource.some( + (r: { name?: string }) => r?.name === name + ); + if (!has) sys.resource.push({ name }); + } + } else if (Array.isArray(sys.resource)) { + // Package absent: the resources do not exist, even if the login + // backend (a GOLD dev stack) has them. + sys.resource = sys.resource.filter( + (r: { name?: string }) => !gold.includes(r?.name ?? '') + ); + } + return route.fulfill({ response: res, json: sys }); + }); + + await page.route('**/api/v2/system/data_masking**', route => + handle(route, state) + ); + + // DreamFactory's own table list for the mocked databases (first run asks + // whether there is anything to scan). Other services pass through. + await page.route(/\/api\/v2\/([^/?]+)\/_table(\?|$)/, route => { + const svc = new URL(route.request().url()).pathname.split('/')[3]; + if (state.unreachable && svc === UNREACHABLE.service) { + return json(route, { error: { message: UNREACHABLE_REASON } }, 500); + } + if (state.builtinDb && svc === BUILTIN_DB.service) { + return json(route, { resource: [] }); + } + const fx = state.fx.services.find(s => s.service === svc); + if (fx) return json(route, { resource: fx.tables }); + return route.fallback(); + }); + await installComplianceMock(page, state); + return state; +} + +/** + * Data Explorer reads of the mocked databases (GET /_table/) + * as df-masking answers them in mode On (decision 20): synthetic rows with + * masked columns masked; a filter condition on a masked column is unknown, + * so the filter (Data Explorer ANDs its column filters) matches no rows; a + * sort on one is ignored. Both are named in X-DreamFactory-Masking-Dropped. + * Outside mode On there are no masking headers and nothing is dropped. + */ +export async function installDataExplorerMaskingMock( + page: Page, + state: Awaited>, + rows = 8 +) { + await page.route(/\/api\/v2\/([^/?]+)\/_table\/([^/?]+)(\?|$)/, route => { + const req = route.request(); + const url = new URL(req.url()); + const [, , , svc, , rawTable] = url.pathname.split('/'); + const table = decodeURIComponent(rawTable ?? ''); + const cols = state.columns.filter( + c => c.service === svc && c.table === table + ); + if (req.method() !== 'GET' || !cols.length) return route.fallback(); + const on = state.mode === 'on'; + const hidden = new Map( + cols + .filter(c => on && c.verdict !== 'clear') + .map(c => [c.column.toLowerCase(), c]) + ); + const dropped: string[] = []; + let none = false; + // Column filters: "(col op value) AND (…)". + for (const part of (url.searchParams.get('filter') ?? '').split(/ AND /i)) { + const m = part.trim().match(/^\(?\s*([^\s()]+)\s+(.+?)\)?$/); + const col = m?.[1].toLowerCase(); + if (!col || !hidden.has(col)) continue; + if (/^IS (NOT )?NULL$/i.test(m![2].trim())) continue; + dropped.push(`filter:${encodeURIComponent(hidden.get(col)!.column)}`); + none = true; + } + const order = (url.searchParams.get('order') ?? '').split(' ')[0]; + if (order && hidden.has(order.toLowerCase())) { + dropped.push(`order:${encodeURIComponent(order)}`); + } + const value = (c: Col, i: number): unknown => { + if (hidden.has(c.column.toLowerCase())) { + const format = c.class ? formatOf(state, c.class) : 'redact'; + return c.class && format + ? sampleMask(c.class, format).after + : applyMask('', 'redact', ''); + } + const t = c.type.toLowerCase(); + if (/int|numeric|serial/.test(t)) return i; + if (t.startsWith('bool')) return i % 2 === 0; + if (t.startsWith('date') || t.startsWith('timestamp')) { + return `2026-0${1 + (i % 9)}-1${i % 10}`; + } + return `${c.column}-${i}`; + }; + const resource = none + ? [] + : Array.from({ length: rows }, (_, i) => + Object.fromEntries(cols.map(c => [c.column, value(c, i + 1)])) + ); + const headers: Record = {}; + if (on && hidden.size) headers['X-DreamFactory-Masking'] = 'on'; + if (dropped.length) { + headers['X-DreamFactory-Masking-Dropped'] = [...new Set(dropped)].join( + ',' + ); + // Same-origin in the app; exposed anyway, like the server's CORS. + headers['Access-Control-Expose-Headers'] = + 'X-DreamFactory-Masking, X-DreamFactory-Masking-Dropped'; + } + return route.fulfill({ + status: 200, + contentType: 'application/json', + headers, + body: JSON.stringify({ resource, meta: { count: resource.length } }), + }); + }); +} + +/** Synthetic sample through a format, for assertions. */ +export const maskSample = (cls: string, f: MaskFormat) => + applyMask(classDef(cls)?.sample ?? '', f, cls); diff --git a/e2e/fixtures/data-masking/fixture.json b/e2e/fixtures/data-masking/fixture.json new file mode 100644 index 000000000..c6046ba51 --- /dev/null +++ b/e2e/fixtures/data-masking/fixture.json @@ -0,0 +1 @@ +{"meta":{"ruleset":"dreamfactory.core","version":"2026.11.0","engine":"spike-0.1.0","generated":"2026-10-28"},"services":[{"service":"crm","label":"CRM database","tables":["access_logs","api_credentials","bank_transfers","chat_messages","clientes","clientes_br","clienti","clients_fr","CustomerContacts","devices","eav_attributes","facilities","hr_staff","klanten","kunden","kunder","legacy_feed","loyalty_members","newsletter_signups","orders_ledger","patient_encounters","payment_methods","products_catalog","service_appointments","staging_import","support_tickets","survey-responses","tbl_cust_master","tenant_settings","user_accounts"]},{"service":"sports","label":"Sports stats","tables":["addresses","affiliation_phases","affiliations","affiliations_documents","affiliations_events","affiliations_media","american_football_action_participants","american_football_action_plays","american_football_defensive_stats","american_football_down_progress_stats","american_football_event_states","american_football_fumbles_stats","american_football_offensive_stats","american_football_passing_stats","american_football_penalties_stats","american_football_rushing_stats","display_names","injury_phases","persons"]},{"service":"store","label":"Web store","tables":["categories","cust_hist","customers","inventory","orderlines","orders","products","reorder"]}],"builtin":[{"class":"national_id","detectors":["Dutch BSN","French NIR","Spanish DNI","Swedish personnummer","UK NI number","US SSN"]},{"class":"tax_id","detectors":["Brazilian CPF","German Steuer-ID","Italian codice fiscale"]},{"class":"passport_number","detectors":["Passport shape"]},{"class":"drivers_license","detectors":["Driver's license shape"]},{"class":"card_number","detectors":["Card checksum (Luhn)","Card shape"]},{"class":"card_security","detectors":["Security code digits"]},{"class":"bank_account","detectors":["Account digits","IBAN checksum"]},{"class":"credential","detectors":["High-entropy secret","JWT","Password hash","Provider API key"]},{"class":"special_category","detectors":["Blood type","ICD-10 code"]},{"class":"email","detectors":["Email syntax"]},{"class":"phone","detectors":["Formatted phone","Phone digits"]},{"class":"person_name","detectors":[]},{"class":"street_address","detectors":["Street pattern"]},{"class":"date_of_birth","detectors":[]},{"class":"ip_address","detectors":["IPv4"]},{"class":"username","detectors":["Username shape"]},{"class":"geo_precise","detectors":["Precise coordinates"]},{"class":"postal_code","detectors":["Dutch postcode","Numeric postal code","UK postcode"]},{"class":"gender","detectors":["Gender values"]},{"class":"age","detectors":["Age range"]},{"class":"card_expiry","detectors":["Expiry date"]},{"class":"financial_personal","detectors":["Money amount"]},{"class":"free_text_pii","detectors":[]}],"columns":[{"id":"crm.access_logs.bytes","service":"crm","table":"access_logs","column":"bytes","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"bytes\" matches no class"]]},{"id":"crm.access_logs.client_ip","service":"crm","table":"access_logs","column":"client_ip","type":"inet","verdict":"clear","class":null,"tier":null,"confidence":30,"total":30,"lines":[[70,"name","Column name \"client_ip\""],[0,"values","0 of 256 values are IPv4 addresses"]]},{"id":"crm.access_logs.latency_ms","service":"crm","table":"access_logs","column":"latency_ms","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"latency_ms\" matches no class"]]},{"id":"crm.access_logs.method","service":"crm","table":"access_logs","column":"method","type":"character varying(7)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"method\" matches no class"]]},{"id":"crm.access_logs.path","service":"crm","table":"access_logs","column":"path","type":"text","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"path\" matches no class"]]},{"id":"crm.access_logs.request_id","service":"crm","table":"access_logs","column":"request_id","type":"uuid","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"request_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.access_logs.server_ip","service":"crm","table":"access_logs","column":"server_ip","type":"inet","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"server_ip\" matches no class"],[0,"context","\"server ip\" rules out ip address"]]},{"id":"crm.access_logs.status","service":"crm","table":"access_logs","column":"status","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"status\" matches no class"]]},{"id":"crm.access_logs.ts","service":"crm","table":"access_logs","column":"ts","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ts\" matches no class"]]},{"id":"crm.access_logs.user_agent","service":"crm","table":"access_logs","column":"user_agent","type":"character varying(200)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"user_agent\" matches no class"]]},{"id":"crm.access_logs.user_email","service":"crm","table":"access_logs","column":"user_email","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"user_email\""],[75,"values","103 of 103 values are valid email addresses"]],"shapes":[["a+@a+.a+",26],["a+.a@a+.a+",17],["a+.a+99@a+.a+",16]],"examples":{"token":"EMAIL_7b601d9c","initial":"c•••••••••••@•••••••.net","redact":"••••••••","null":null}},{"id":"crm.access_logs.x_forwarded_for","service":"crm","table":"access_logs","column":"x_forwarded_for","type":"character varying(120)","verdict":"clear","class":null,"tier":null,"confidence":30,"total":30,"lines":[[70,"name","Column name \"x_forwarded_for\""],[0,"values","0 of 256 values are IPv4 addresses"]]},{"id":"crm.api_credentials.api_key","service":"crm","table":"api_credentials","column":"api_key","type":"character varying(110)","verdict":"masked","class":"credential","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"api_key\""],[75,"values","256 of 256 values are provider API keys"]],"shapes":[["a+_a+_a+9a+9a+",1],["a+_a+_9Aa+9Aa+",0],["a+_a+_Aa+99a+9a+9a+",0]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.api_credentials.aws_access_key_id","service":"crm","table":"api_credentials","column":"aws_access_key_id","type":"character varying(20)","verdict":"masked","class":"credential","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"aws_access_key_id\""],[0,"context","\"key id\" rules out credential"],[75,"values","256 of 256 values are provider API keys"]],"shapes":[["A+9A+9A+",14],["A+9A+",9],["A+9A+9A+9A+",4]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.api_credentials.aws_secret_access_key","service":"crm","table":"api_credentials","column":"aws_secret_access_key","type":"character varying(40)","verdict":"masked","class":"credential","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"aws_secret_access_key\""],[20,"values","256 of 256 values look like secrets"]],"shapes":[["+a+9A9a9a++9999",0],["+a+a+9Aa+9Aa++Aa+9A/A9a+9A+/Aa+",0],["/9+99Aa+9A9a9/a9Aa+/a+",0]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.api_credentials.client_id","service":"crm","table":"api_credentials","column":"client_id","type":"uuid","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"client_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.api_credentials.config_checksum","service":"crm","table":"api_credentials","column":"config_checksum","type":"character(32)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"config_checksum\" matches no class"]]},{"id":"crm.api_credentials.created_by_user_id","service":"crm","table":"api_credentials","column":"created_by_user_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"created_by_user_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.api_credentials.expires_at","service":"crm","table":"api_credentials","column":"expires_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"expires_at\" matches no class"]]},{"id":"crm.api_credentials.id","service":"crm","table":"api_credentials","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.api_credentials.integration_name","service":"crm","table":"api_credentials","column":"integration_name","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"integration_name\""],[0,"values","0% of values are known names"]]},{"id":"crm.api_credentials.key_fingerprint","service":"crm","table":"api_credentials","column":"key_fingerprint","type":"character varying(60)","verdict":"masked","class":"credential","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"key_fingerprint\""],[0,"context","\"key fingerprint\" rules out health & special category"],[20,"values","256 of 256 values look like secrets"]],"shapes":[["A+999:999Aa+/99A99a9a+9Aa+/a+99A",0],["A+999:99A+9a+9Aa+/Aa+9a9a+9Aa+",0],["A+999:99a++Aa+99a9Aa+/9",0]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.api_credentials.last_deploy_commit","service":"crm","table":"api_credentials","column":"last_deploy_commit","type":"character(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"last_deploy_commit\" matches no class"]]},{"id":"crm.api_credentials.refresh_token","service":"crm","table":"api_credentials","column":"refresh_token","type":"text","verdict":"masked","class":"credential","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"refresh_token\""],[75,"values","194 of 194 values are JSON web tokens"]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.api_credentials.revoked","service":"crm","table":"api_credentials","column":"revoked","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"revoked\" matches no class"]]},{"id":"crm.api_credentials.scopes","service":"crm","table":"api_credentials","column":"scopes","type":"character varying(120)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"scopes\" matches no class"]]},{"id":"crm.api_credentials.webhook_signing_secret","service":"crm","table":"api_credentials","column":"webhook_signing_secret","type":"character varying(50)","verdict":"masked","class":"credential","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"webhook_signing_secret\""],[20,"values","256 of 256 values look like secrets"]],"shapes":[["a+_Aa+9a+",1],["a+_Aa+9a+9Aa+",1],["a+_Aa+9a+9a+",1]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.bank_transfers.amount","service":"crm","table":"bank_transfers","column":"amount","type":"numeric(12,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"amount\" matches no class"]]},{"id":"crm.bank_transfers.batch_id","service":"crm","table":"bank_transfers","column":"batch_id","type":"uuid","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"batch_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.bank_transfers.beneficiary_iban","service":"crm","table":"bank_transfers","column":"beneficiary_iban","type":"character varying(42)","verdict":"masked","class":"bank_account","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"beneficiary_iban\""],[75,"values","212 of 212 values are valid IBANs"]],"shapes":[["A+9999999999999999999999",14],["A+99 9999 9999 9999 9999 9999 999",13],["A+9999999999999999999999999",12]],"examples":{"last4":"••••••••••••••••••••7246","redact":"••••••••","null":null}},{"id":"crm.bank_transfers.beneficiary_name","service":"crm","table":"bank_transfers","column":"beneficiary_name","type":"character varying(100)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"beneficiary_name\" in a person table"],[45,"values","99% of values are known names"]],"shapes":[["Aa+ Aa+",90],["Aa+ a+ Aa+",6],["Aa+ A'Aa+",2]],"examples":{"token":"NAME_a0a8b460","initial":"C••••••• D•••","redact":"••••••••","null":null}},{"id":"crm.bank_transfers.bic","service":"crm","table":"bank_transfers","column":"bic","type":"character varying(11)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"bic\" matches no class"]]},{"id":"crm.bank_transfers.currency","service":"crm","table":"bank_transfers","column":"currency","type":"character(3)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"currency\" matches no class"]]},{"id":"crm.bank_transfers.id_no","service":"crm","table":"bank_transfers","column":"id_no","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"id_no\""],[0,"values","57 of 256 values are 9-digit SSN candidates"]]},{"id":"crm.bank_transfers.id_type","service":"crm","table":"bank_transfers","column":"id_type","type":"character varying(16)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id_type\" matches no class"]]},{"id":"crm.bank_transfers.internal_ref","service":"crm","table":"bank_transfers","column":"internal_ref","type":"character varying(34)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"internal_ref\" matches no class"]]},{"id":"crm.bank_transfers.remittance_info","service":"crm","table":"bank_transfers","column":"remittance_info","type":"character varying(140)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"remittance_info\" matches no class"]]},{"id":"crm.bank_transfers.transfer_id","service":"crm","table":"bank_transfers","column":"transfer_id","type":"bigint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"transfer_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.bank_transfers.us_account_no","service":"crm","table":"bank_transfers","column":"us_account_no","type":"character varying(17)","verdict":"masked","class":"bank_account","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"us_account_no\""],[20,"values","44 of 44 values look like account numbers"]],"shapes":[["99999999999",29],["999999999",27],["99999999",20]],"examples":{"last4":"•••••••0535","redact":"••••••••","null":null}},{"id":"crm.bank_transfers.us_routing_aba","service":"crm","table":"bank_transfers","column":"us_routing_aba","type":"character(9)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"us_routing_aba\" matches no class"]]},{"id":"crm.bank_transfers.value_date","service":"crm","table":"bank_transfers","column":"value_date","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"value_date\" matches no class"]]},{"id":"crm.chat_messages.conversation_id","service":"crm","table":"chat_messages","column":"conversation_id","type":"uuid","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"conversation_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.chat_messages.message_id","service":"crm","table":"chat_messages","column":"message_id","type":"bigint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"message_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.chat_messages.metadata","service":"crm","table":"chat_messages","column":"metadata","type":"jsonb","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"metadata\" matches no class"]]},{"id":"crm.chat_messages.payload","service":"crm","table":"chat_messages","column":"payload","type":"jsonb","verdict":"masked","class":"free_text_pii","tier":"free_text","confidence":75,"total":75,"lines":[[0,"name","Column name \"payload\" matches no class"],[75,"text","Personal data inside the text in 144 rows"]],"shapes":[["{\"a+\": \"Aa+!\", \"a+\": []}",13],["{\"a+\": \"Aa+ a+ a+ a+.\", \"a+\": []}",10],["{\"a+\": \"Aa+ a+ a+ a+ a+ a+.\", \"a+\": []}",8]],"examples":{"span":"Call Jane at •••••• or ••••••","redact":"••••••••","null":null}},{"id":"crm.chat_messages.sender_role","service":"crm","table":"chat_messages","column":"sender_role","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sender_role\" matches no class"]]},{"id":"crm.chat_messages.sent_at","service":"crm","table":"chat_messages","column":"sent_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sent_at\" matches no class"]]},{"id":"crm.clientes.apellido1","service":"crm","table":"clientes","column":"apellido1","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"apellido1\""],[45,"values","100% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_5491d568","initial":"F••••••••","redact":"••••••••","null":null}},{"id":"crm.clientes.codigo_postal","service":"crm","table":"clientes","column":"codigo_postal","type":"character(5)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"codigo_postal\""],[20,"values","256 of 256 values look like postal codes"]],"shapes":[["99999",100]],"examples":{"redact":"••••••••","token":"ZIP_09da7b61","null":null}},{"id":"crm.clientes.correo","service":"crm","table":"clientes","column":"correo","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"correo\""],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",30],["a+99@a+.a+",16],["a+.a+@a+.a+",15]],"examples":{"token":"EMAIL_7b24fa3b","initial":"c•••••••••@•••••.com","redact":"••••••••","null":null}},{"id":"crm.clientes.direccion","service":"crm","table":"clientes","column":"direccion","type":"character varying(120)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"direccion\""],[45,"values","231 of 256 values look like street addresses"]],"shapes":[["Aa+ a+ Aa+ 99, 9L+ A",25],["Aa+ Aa+ 99, 9L+ A",18],["Aa+ a+ Aa+ 999, 9L+ A",14]],"examples":{"redact":"••••••••","token":"ADDR_f57bc71c","null":null}},{"id":"crm.clientes.dni","service":"crm","table":"clientes","column":"dni","type":"character(9)","verdict":"masked","class":"national_id","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"dni\""],[75,"values","256 of 256 values are valid Spanish DNIs"]],"shapes":[["99999999A",100]],"examples":{"redact":"••••••••","last4":"•••••989C","null":null}},{"id":"crm.clientes.estado","service":"crm","table":"clientes","column":"estado","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"estado\" matches no class"]]},{"id":"crm.clientes.fecha_alta","service":"crm","table":"clientes","column":"fecha_alta","type":"date","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"fecha_alta\" matches no class"],[45,"values","Values look like birth dates (17 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"2022-••-••","redact":"••••••••","token":"DOB_d46be7e3","null":null}},{"id":"crm.clientes.fecha_nacimiento","service":"crm","table":"clientes","column":"fecha_nacimiento","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"fecha_nacimiento\""],[45,"values","Values look like birth dates (64 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1966-••-••","redact":"••••••••","token":"DOB_1219dd3a","null":null}},{"id":"crm.clientes.id_cliente","service":"crm","table":"clientes","column":"id_cliente","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id_cliente\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.clientes.nacionalidad","service":"crm","table":"clientes","column":"nacionalidad","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"nacionalidad\" matches no class"]]},{"id":"crm.clientes.nombre","service":"crm","table":"clientes","column":"nombre","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"nombre\" in a person table"],[45,"values","97% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_88d6e9d6","initial":"C•••••","redact":"••••••••","null":null}},{"id":"crm.clientes.provincia","service":"crm","table":"clientes","column":"provincia","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"provincia\" matches no class"]]},{"id":"crm.clientes.segundo_apellido","service":"crm","table":"clientes","column":"segundo_apellido","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"segundo_apellido\""],[45,"values","100% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_8481c26a","initial":"M•••••","redact":"••••••••","null":null}},{"id":"crm.clientes.sexo","service":"crm","table":"clientes","column":"sexo","type":"character(1)","verdict":"masked","class":"gender","tier":"quasi","confidence":80,"total":80,"lines":[[70,"name","Column name \"sexo\""],[10,"values","256 of 256 values are gender values (few distinct values)"]],"shapes":[["A",100]],"examples":{"redact":"••••••••","token":"GENDER_ed07f0c6","null":null}},{"id":"crm.clientes.telefono","service":"crm","table":"clientes","column":"telefono","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"telefono\""],[35,"values","195 of 256 values are formatted phone numbers"]],"shapes":[["+99 999 999 999",39],["+99 999 99 99 99",36],["999999999",23]],"examples":{"token":"PHONE_f403d6f5","last4":"+•• ••• •• 49 08","redact":"••••••••","null":null}},{"id":"crm.clientes_br.bairro","service":"crm","table":"clientes_br","column":"bairro","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"bairro\" matches no class"]]},{"id":"crm.clientes_br.celular","service":"crm","table":"clientes_br","column":"celular","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":90,"total":90,"lines":[[70,"name","Column name \"celular\""],[20,"values","131 of 256 values are formatted phone numbers"]],"shapes":[["+99 99 99999-9999",51],["(99) 99999-9999",48]],"examples":{"token":"PHONE_aed042fb","last4":"+•• •• •••••-3153","redact":"••••••••","null":null}},{"id":"crm.clientes_br.cep","service":"crm","table":"clientes_br","column":"cep","type":"character(9)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":70,"total":70,"lines":[[70,"name","Column name \"cep\""],[0,"values","0 of 256 values are US ZIP+4 codes"]],"shapes":[["99999-999",100]],"examples":{"redact":"••••••••","token":"ZIP_b33f5551","null":null}},{"id":"crm.clientes_br.cidade","service":"crm","table":"clientes_br","column":"cidade","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"cidade\" matches no class"]]},{"id":"crm.clientes_br.cpf","service":"crm","table":"clientes_br","column":"cpf","type":"character varying(14)","verdict":"masked","class":"tax_id","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"cpf\""],[75,"values","256 of 256 values are valid Brazilian CPFs"]],"shapes":[["999.999.999-99",80],["99999999999",19]],"examples":{"redact":"••••••••","last4":"•••.•••.•59-99","null":null}},{"id":"crm.clientes_br.criado_em","service":"crm","table":"clientes_br","column":"criado_em","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"criado_em\" matches no class"]]},{"id":"crm.clientes_br.data_de_nascimento","service":"crm","table":"clientes_br","column":"data_de_nascimento","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"data_de_nascimento\""],[45,"values","Values look like birth dates (65 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1964-••-••","redact":"••••••••","token":"DOB_b9e2c895","null":null}},{"id":"crm.clientes_br.email","service":"crm","table":"clientes_br","column":"email","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"email\""],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",14],["a+@a+.a+.a+",14],["a+.a@a+.a+.a+",10]],"examples":{"token":"EMAIL_97823596","initial":"g••••••••••••@•••••••.com","redact":"••••••••","null":null}},{"id":"crm.clientes_br.endereco","service":"crm","table":"clientes_br","column":"endereco","type":"character varying(120)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"endereco\""],[45,"values","256 of 256 values look like street addresses"]],"shapes":[["Aa+ Aa+, 9999",14],["Aa+ a+ Aa+, 9999",11],["Aa+ a+ Aa+, 999",7]],"examples":{"redact":"••••••••","token":"ADDR_950c9e78","null":null}},{"id":"crm.clientes_br.estado_civil","service":"crm","table":"clientes_br","column":"estado_civil","type":"character varying(16)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"estado_civil\" matches no class"]]},{"id":"crm.clientes_br.id","service":"crm","table":"clientes_br","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.clientes_br.nome_completo","service":"crm","table":"clientes_br","column":"nome_completo","type":"character varying(120)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"nome_completo\""],[45,"values","100% of values are known names"]],"shapes":[["Aa+ Aa+ Aa+",59],["Aa+ Aa+",40]],"examples":{"token":"NAME_bca2003e","initial":"B•••• G••••","redact":"••••••••","null":null}},{"id":"crm.clientes_br.raca_cor","service":"crm","table":"clientes_br","column":"raca_cor","type":"character varying(16)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"raca_cor\" matches no class"]]},{"id":"crm.clientes_br.renda_mensal","service":"crm","table":"clientes_br","column":"renda_mensal","type":"numeric(10,2)","verdict":"masked","class":"financial_personal","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"renda_mensal\" in a person table"],[20,"values","256 of 256 values are money amounts"]],"shapes":[["99999.99",67],["9999.99",32]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.clientes_br.rg","service":"crm","table":"clientes_br","column":"rg","type":"character varying(14)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rg\" matches no class"]]},{"id":"crm.clientes_br.uf","service":"crm","table":"clientes_br","column":"uf","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"uf\" matches no class"]]},{"id":"crm.clienti.cap","service":"crm","table":"clienti","column":"cap","type":"character(5)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"cap\""],[20,"values","256 of 256 values look like postal codes"]],"shapes":[["99999",100]],"examples":{"redact":"••••••••","token":"ZIP_2059afae","null":null}},{"id":"crm.clienti.cellulare","service":"crm","table":"clienti","column":"cellulare","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"cellulare\""],[45,"values","242 of 256 values are formatted phone numbers"]],"shapes":[["+999999999999",41],["999-9999999",30],["+99 999 9999999",28]],"examples":{"token":"PHONE_803f2817","last4":"•••-•••5806","redact":"••••••••","null":null}},{"id":"crm.clienti.citta","service":"crm","table":"clienti","column":"citta","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"citta\" matches no class"]]},{"id":"crm.clienti.codice_cliente","service":"crm","table":"clienti","column":"codice_cliente","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"codice_cliente\" matches no class"]]},{"id":"crm.clienti.codice_fiscale","service":"crm","table":"clienti","column":"codice_fiscale","type":"character(16)","verdict":"masked","class":"tax_id","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"codice_fiscale\""],[75,"values","256 of 256 values are valid Italian tax codes"]],"shapes":[["A+99A99A999A",100]],"examples":{"redact":"••••••••","last4":"••••••••••••273X","null":null}},{"id":"crm.clienti.cognome","service":"crm","table":"clienti","column":"cognome","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"cognome\""],[45,"values","95% of values are known names"]],"shapes":[["Aa+",91],["Aa+ Aa+",4],["A'Aa+",3]],"examples":{"token":"NAME_7cdb76b3","initial":"R••••","redact":"••••••••","null":null}},{"id":"crm.clienti.data_iscrizione","service":"crm","table":"clienti","column":"data_iscrizione","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"data_iscrizione\" matches no class"]]},{"id":"crm.clienti.data_nascita","service":"crm","table":"clienti","column":"data_nascita","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"data_nascita\""],[45,"values","Values look like birth dates (65 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1982-••-••","redact":"••••••••","token":"DOB_bfbe5cb6","null":null}},{"id":"crm.clienti.iban","service":"crm","table":"clienti","column":"iban","type":"character varying(34)","verdict":"masked","class":"bank_account","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"iban\""],[75,"values","171 of 171 values are valid IBANs"]],"shapes":[["A+99A9999999999999999999999",61],["A+99 A999 9999 9999 9999 9999 999",38]],"examples":{"last4":"•••••••••••••••••••••••7549","redact":"••••••••","null":null}},{"id":"crm.clienti.indirizzo","service":"crm","table":"clienti","column":"indirizzo","type":"character varying(120)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"indirizzo\""],[45,"values","256 of 256 values look like street addresses"]],"shapes":[["Aa+ Aa+ 999",28],["Aa+ Aa+ 99",25],["Aa+ a+ Aa+ 99",14]],"examples":{"redact":"••••••••","token":"ADDR_eeb8d816","null":null}},{"id":"crm.clienti.luogo_nascita","service":"crm","table":"clienti","column":"luogo_nascita","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"luogo_nascita\" matches no class"]]},{"id":"crm.clienti.nome","service":"crm","table":"clienti","column":"nome","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"nome\" in a person table"],[45,"values","100% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_dc39d57d","initial":"E••••","redact":"••••••••","null":null}},{"id":"crm.clienti.pec","service":"crm","table":"clienti","column":"pec","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":75,"total":75,"lines":[[0,"name","Column name \"pec\" matches no class"],[75,"values","131 of 131 values are valid email addresses"]],"shapes":[["a+@a+.a+",16],["a+.a+@a+.a+",12],["a+_a+99@a+.a+",9]],"examples":{"token":"EMAIL_025fee13","initial":"e••••••••••@•••.it","redact":"••••••••","null":null}},{"id":"crm.clienti.provincia","service":"crm","table":"clienti","column":"provincia","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"provincia\" matches no class"]]},{"id":"crm.clienti.reddito_annuo","service":"crm","table":"clienti","column":"reddito_annuo","type":"numeric(12,2)","verdict":"masked","class":"financial_personal","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"reddito_annuo\" in a person table"],[20,"values","166 of 166 values are money amounts"]],"shapes":[["99999.99",63],["999999.99",36]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.clients_fr.adresse","service":"crm","table":"clients_fr","column":"adresse","type":"character varying(120)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"adresse\" in a person table"],[45,"values","256 of 256 values look like street addresses"]],"shapes":[["99 a+ a+ a+ Aa+",22],["99 a+ Aa+ Aa+",21],["99 a+ a+ Aa+",19]],"examples":{"redact":"••••••••","token":"ADDR_7e98a007","null":null}},{"id":"crm.clients_fr.civilite","service":"crm","table":"clients_fr","column":"civilite","type":"character varying(4)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"civilite\" matches no class"]]},{"id":"crm.clients_fr.code_postal","service":"crm","table":"clients_fr","column":"code_postal","type":"character(5)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"code_postal\""],[20,"values","256 of 256 values look like postal codes"]],"shapes":[["99999",100]],"examples":{"redact":"••••••••","token":"ZIP_581c89b8","null":null}},{"id":"crm.clients_fr.courriel","service":"crm","table":"clients_fr","column":"courriel","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"courriel\""],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",27],["a+.a@a+.a+",16],["a+.a+99@a+.a+",15]],"examples":{"token":"EMAIL_581b18ac","initial":"f••••••••@•••••••.net","redact":"••••••••","null":null}},{"id":"crm.clients_fr.date_creation","service":"crm","table":"clients_fr","column":"date_creation","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"date_creation\" matches no class"],[0,"context","The name says this is another kind of date"]]},{"id":"crm.clients_fr.date_naissance","service":"crm","table":"clients_fr","column":"date_naissance","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"date_naissance\""],[45,"values","Values look like birth dates (65 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1976-••-••","redact":"••••••••","token":"DOB_7b0dbf38","null":null}},{"id":"crm.clients_fr.id_client","service":"crm","table":"clients_fr","column":"id_client","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id_client\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.clients_fr.nom","service":"crm","table":"clients_fr","column":"nom","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"nom\" in a person table"],[45,"values","96% of values are known names"]],"shapes":[["Aa+",56],["A+",40],["A'A+",1]],"examples":{"token":"NAME_39bed45a","initial":"F•••••••","redact":"••••••••","null":null}},{"id":"crm.clients_fr.num_secu","service":"crm","table":"clients_fr","column":"num_secu","type":"character varying(21)","verdict":"masked","class":"national_id","tier":"restricted","confidence":75,"total":75,"lines":[[0,"name","Column name \"num_secu\" matches no class"],[75,"values","256 of 256 values are valid French NIRs"]],"shapes":[["999999999999999",58],["9 99 99 99 999 999 99",41]],"examples":{"redact":"••••••••","last4":"• •• •• •• ••• •49 51","null":null}},{"id":"crm.clients_fr.prenom","service":"crm","table":"clients_fr","column":"prenom","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"prenom\""],[45,"values","89% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_a6f56a60","initial":"F••••","redact":"••••••••","null":null}},{"id":"crm.clients_fr.situation_familiale","service":"crm","table":"clients_fr","column":"situation_familiale","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"situation_familiale\" matches no class"]]},{"id":"crm.clients_fr.telephone","service":"crm","table":"clients_fr","column":"telephone","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"telephone\""],[35,"values","162 of 256 values are formatted phone numbers"]],"shapes":[["9999999999",36],["99 99 99 99 99",32],["+99 9 99 99 99 99",30]],"examples":{"token":"PHONE_dc9e3314","last4":"+•• • •• •• 12 30","redact":"••••••••","null":null}},{"id":"crm.clients_fr.ville","service":"crm","table":"clients_fr","column":"ville","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ville\" matches no class"]]},{"id":"crm.CustomerContacts.AccountName","service":"crm","table":"CustomerContacts","column":"AccountName","type":"character varying(80)","verdict":"masked","class":"username","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"AccountName\" in a person table"],[0,"context","\"account name\" rules out person name"],[0,"values","0 of 256 values look like usernames"]],"shapes":[["Aa+ Aa+",93],["Aa+ Aa+ Aa+",6]],"examples":{"token":"USER_7a289817","initial":"C•••••••• L•••","redact":"••••••••","null":null}},{"id":"crm.CustomerContacts.Birthdate","service":"crm","table":"CustomerContacts","column":"Birthdate","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"Birthdate\""],[45,"values","Values look like birth dates (59 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1955-••-••","redact":"••••••••","token":"DOB_32f4f15a","null":null}},{"id":"crm.CustomerContacts.ContactID","service":"crm","table":"CustomerContacts","column":"ContactID","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ContactID\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.CustomerContacts.CreatedDate","service":"crm","table":"CustomerContacts","column":"CreatedDate","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"CreatedDate\" matches no class"]]},{"id":"crm.CustomerContacts.EMail","service":"crm","table":"CustomerContacts","column":"EMail","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"EMail\""],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",28],["a+.a@a+.a+",19],["a+.a+99@a+.a+",14]],"examples":{"token":"EMAIL_421ec9ba","initial":"p••••••••••••@••••••••••.com","redact":"••••••••","null":null}},{"id":"crm.CustomerContacts.FirstName","service":"crm","table":"CustomerContacts","column":"FirstName","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"FirstName\""],[45,"values","90% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_2c6d880b","initial":"P••••","redact":"••••••••","null":null}},{"id":"crm.CustomerContacts.JobTitle","service":"crm","table":"CustomerContacts","column":"JobTitle","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"JobTitle\" matches no class"]]},{"id":"crm.CustomerContacts.LastName","service":"crm","table":"CustomerContacts","column":"LastName","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"LastName\""],[45,"values","81% of values are known names"]],"shapes":[["Aa+",98],["A'Aa+",1]],"examples":{"token":"NAME_08a3d990","initial":"L••••","redact":"••••••••","null":null}},{"id":"crm.CustomerContacts.LeadSource","service":"crm","table":"CustomerContacts","column":"LeadSource","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"LeadSource\" matches no class"]]},{"id":"crm.CustomerContacts.MailingCity","service":"crm","table":"CustomerContacts","column":"MailingCity","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"MailingCity\" matches no class"]]},{"id":"crm.CustomerContacts.MailingCountry","service":"crm","table":"CustomerContacts","column":"MailingCountry","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"MailingCountry\" matches no class"]]},{"id":"crm.CustomerContacts.MailingPostalCode","service":"crm","table":"CustomerContacts","column":"MailingPostalCode","type":"character varying(10)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"MailingPostalCode\""],[20,"values","206 of 256 values look like postal codes"]],"shapes":[["99999",80],["99999-9999",19]],"examples":{"redact":"••••••••","token":"ZIP_40693423","null":null}},{"id":"crm.CustomerContacts.MailingStreet","service":"crm","table":"CustomerContacts","column":"MailingStreet","type":"character varying(120)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"MailingStreet\""],[45,"values","256 of 256 values look like street addresses"]],"shapes":[["9999 Aa+ Aa+",52],["9999 Aa+ Aa+ Aa+ 99",23],["9999 Aa+ Aa+ Aa+ 9",8]],"examples":{"redact":"••••••••","token":"ADDR_310c6145","null":null}},{"id":"crm.CustomerContacts.MobilePhone","service":"crm","table":"CustomerContacts","column":"MobilePhone","type":"character varying(25)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"MobilePhone\""],[45,"values","210 of 210 values are formatted phone numbers"]],"shapes":[["999.999.9999",23],["999-999-9999",21],["+99999999999",20]],"examples":{"token":"PHONE_7c722a6d","last4":"•••.•••.5530","redact":"••••••••","null":null}},{"id":"crm.CustomerContacts.Notes","service":"crm","table":"CustomerContacts","column":"Notes","type":"text","verdict":"masked","class":"free_text_pii","tier":"free_text","confidence":75,"total":75,"lines":[[0,"name","Column name \"Notes\" matches no class"],[75,"text","Personal data inside the text in 10 rows"]],"shapes":[["Aa+ a+ a+ a+ a+; a+ a+.",10],["Aa+ a+ a+, a+ a+ a+ a+ a+.",10],["Aa+ a+ a+ a+.",10]],"examples":{"span":"Prefers personal email •••••• for contracts.","redact":"••••••••","null":null}},{"id":"crm.CustomerContacts.OwnerId","service":"crm","table":"CustomerContacts","column":"OwnerId","type":"character varying(18)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"OwnerId\" matches no class"]]},{"id":"crm.CustomerContacts.WorkPhone","service":"crm","table":"CustomerContacts","column":"WorkPhone","type":"character varying(30)","verdict":"masked","class":"phone","tier":"identity","confidence":90,"total":90,"lines":[[70,"name","Column name \"WorkPhone\""],[20,"values","106 of 186 values are formatted phone numbers"]],"shapes":[["(999) 999-9999",56],["(999) 999-9999 a9999",22],["(999) 999-9999 a+. 999",17]],"examples":{"token":"PHONE_e0fa6b70","last4":"(•••) •••-5142","redact":"••••••••","null":null}},{"id":"crm.devices.asset_tag","service":"crm","table":"devices","column":"asset_tag","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"asset_tag\" matches no class"]]},{"id":"crm.devices.assigned_emp_no","service":"crm","table":"devices","column":"assigned_emp_no","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"assigned_emp_no\" matches no class"]]},{"id":"crm.devices.device_id","service":"crm","table":"devices","column":"device_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"device_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.devices.device_type","service":"crm","table":"devices","column":"device_type","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"device_type\" matches no class"]]},{"id":"crm.devices.facility_id","service":"crm","table":"devices","column":"facility_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"facility_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.devices.firmware_version","service":"crm","table":"devices","column":"firmware_version","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"firmware_version\" matches no class"]]},{"id":"crm.devices.hostname","service":"crm","table":"devices","column":"hostname","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"hostname\" matches no class"],[0,"context","\"host name\" rules out person name"]]},{"id":"crm.devices.imei","service":"crm","table":"devices","column":"imei","type":"character varying(15)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"imei\" matches no class"]]},{"id":"crm.devices.last_heartbeat","service":"crm","table":"devices","column":"last_heartbeat","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"last_heartbeat\" matches no class"]]},{"id":"crm.devices.mac_address","service":"crm","table":"devices","column":"mac_address","type":"character varying(17)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"mac_address\" matches no class"],[0,"context","\"mac address\" rules out street address"]]},{"id":"crm.devices.mgmt_ip","service":"crm","table":"devices","column":"mgmt_ip","type":"character varying(15)","verdict":"masked","class":"ip_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"mgmt_ip\""],[45,"values","256 of 256 values are IPv4 addresses"]],"shapes":[["99.99.999.999",15],["99.99.99.999",11],["99.99.999.99",10]],"examples":{"token":"IP_3b37a1f9","redact":"••••••••","null":null}},{"id":"crm.devices.serial_number","service":"crm","table":"devices","column":"serial_number","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"serial_number\" matches no class"]]},{"id":"crm.devices.sim_iccid","service":"crm","table":"devices","column":"sim_iccid","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sim_iccid\" matches no class"]]},{"id":"crm.eav_attributes.attribute_name","service":"crm","table":"eav_attributes","column":"attribute_name","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"attribute_name\" matches no class"],[0,"context","\"attribute name\" rules out person name"]]},{"id":"crm.eav_attributes.attribute_value","service":"crm","table":"eav_attributes","column":"attribute_value","type":"character varying(200)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"attribute_value\" matches no class"]]},{"id":"crm.eav_attributes.entity_id","service":"crm","table":"eav_attributes","column":"entity_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"entity_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.eav_attributes.entity_type","service":"crm","table":"eav_attributes","column":"entity_type","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"entity_type\" matches no class"]]},{"id":"crm.eav_attributes.id","service":"crm","table":"eav_attributes","column":"id","type":"bigint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.eav_attributes.updated_at","service":"crm","table":"eav_attributes","column":"updated_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"updated_at\" matches no class"]]},{"id":"crm.facilities.city","service":"crm","table":"facilities","column":"city","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"city\" matches no class"]]},{"id":"crm.facilities.facility_id","service":"crm","table":"facilities","column":"facility_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"facility_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.facilities.facility_name","service":"crm","table":"facilities","column":"facility_name","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"facility_name\""],[0,"values","0% of values are known names"]]},{"id":"crm.facilities.facility_type","service":"crm","table":"facilities","column":"facility_type","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"facility_type\" matches no class"]]},{"id":"crm.facilities.latitude","service":"crm","table":"facilities","column":"latitude","type":"numeric(9,6)","verdict":"review","class":"geo_precise","tier":"identity","confidence":60,"total":60,"lines":[[15,"name","Column name \"latitude\""],[45,"values","220 of 220 values are precise coordinates"]],"shapes":[["99.999999",100]],"examples":{"redact":"••••••••","token":"GEO_fe60ac5a","null":null}},{"id":"crm.facilities.longitude","service":"crm","table":"facilities","column":"longitude","type":"numeric(9,6)","verdict":"review","class":"geo_precise","tier":"identity","confidence":60,"total":60,"lines":[[15,"name","Column name \"longitude\""],[45,"values","220 of 220 values are precise coordinates"]],"shapes":[["-99.999999",66],["-999.999999",33]],"examples":{"redact":"••••••••","token":"GEO_654c5a12","null":null}},{"id":"crm.facilities.main_phone","service":"crm","table":"facilities","column":"main_phone","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"main_phone\""],[45,"values","220 of 220 values are formatted phone numbers"]],"shapes":[["(999) 999-9999",100]],"examples":{"token":"PHONE_f67b93a8","last4":"(•••) •••-5645","redact":"••••••••","null":null}},{"id":"crm.facilities.manager_email","service":"crm","table":"facilities","column":"manager_email","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"manager_email\""],[75,"values","220 of 220 values are valid email addresses"]],"shapes":[["a+@a+.a+",31],["a+.a@a+.a+",17],["a+.a+99@a+.a+",12]],"examples":{"token":"EMAIL_86bac880","initial":"b••••••••••••••@••••••••••••••••••.com","redact":"••••••••","null":null}},{"id":"crm.facilities.manager_name","service":"crm","table":"facilities","column":"manager_name","type":"character varying(80)","verdict":"review","class":"person_name","tier":"identity","confidence":60,"total":60,"lines":[[15,"name","Column name \"manager_name\""],[45,"values","96% of values are known names"]],"shapes":[["Aa+ Aa+",97],["Aa+ A'Aa+",2]],"examples":{"token":"NAME_052484ad","initial":"B•••••• T•••••••","redact":"••••••••","null":null}},{"id":"crm.facilities.opened_on","service":"crm","table":"facilities","column":"opened_on","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"opened_on\" matches no class"],[0,"context","The name says this is another kind of date"]]},{"id":"crm.facilities.operator_duns","service":"crm","table":"facilities","column":"operator_duns","type":"character(9)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"operator_duns\" matches no class"]]},{"id":"crm.facilities.postal_code","service":"crm","table":"facilities","column":"postal_code","type":"character varying(10)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"postal_code\""],[20,"values","220 of 220 values look like postal codes"]],"shapes":[["99999",100]],"examples":{"redact":"••••••••","token":"ZIP_02c3f45e","null":null}},{"id":"crm.facilities.region","service":"crm","table":"facilities","column":"region","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"region\" matches no class"]]},{"id":"crm.facilities.sq_ft","service":"crm","table":"facilities","column":"sq_ft","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sq_ft\" matches no class"]]},{"id":"crm.facilities.street_address","service":"crm","table":"facilities","column":"street_address","type":"character varying(120)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"street_address\""],[45,"values","220 of 220 values look like street addresses"]],"shapes":[["99999 Aa+ Aa+",61],["9999 Aa+ Aa+",33],["999 Aa+ Aa+",5]],"examples":{"redact":"••••••••","token":"ADDR_6d77731e","null":null}},{"id":"crm.facilities.wifi_password","service":"crm","table":"facilities","column":"wifi_password","type":"character varying(40)","verdict":"masked","class":"credential","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"wifi_password\""],[0,"values","0 of 220 values are password hashes"]],"shapes":[["Aa+!9999",5],["Aa+#9999a+",5],["Aa+_9999Aa+",4]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.facilities.wifi_ssid","service":"crm","table":"facilities","column":"wifi_ssid","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"wifi_ssid\" matches no class"]]},{"id":"crm.hr_staff.background_check_note","service":"crm","table":"hr_staff","column":"background_check_note","type":"text","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"background_check_note\" matches no class"]]},{"id":"crm.hr_staff.badge_id","service":"crm","table":"hr_staff","column":"badge_id","type":"character varying(8)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"badge_id\" matches no class"]]},{"id":"crm.hr_staff.bonus_pct","service":"crm","table":"hr_staff","column":"bonus_pct","type":"numeric(4,1)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"bonus_pct\" matches no class"]]},{"id":"crm.hr_staff.cost_center","service":"crm","table":"hr_staff","column":"cost_center","type":"character varying(6)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"cost_center\" matches no class"]]},{"id":"crm.hr_staff.d_o_b","service":"crm","table":"hr_staff","column":"d_o_b","type":"date","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"d_o_b\" matches no class"],[45,"values","Values look like birth dates (64 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1997-••-••","redact":"••••••••","token":"DOB_debabc33","null":null}},{"id":"crm.hr_staff.dept","service":"crm","table":"hr_staff","column":"dept","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"dept\" matches no class"]]},{"id":"crm.hr_staff.disability_status","service":"crm","table":"hr_staff","column":"disability_status","type":"character varying(10)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"disability_status\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes (few distinct values)"]],"shapes":[["Aa+",100]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.hr_staff.emp_no","service":"crm","table":"hr_staff","column":"emp_no","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"emp_no\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.hr_staff.ethnicity","service":"crm","table":"hr_staff","column":"ethnicity","type":"character varying(45)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"ethnicity\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes"]],"shapes":[["Aa+ a+ Aa+ Aa+",28],["Aa+",27],["Aa+ a+ a+",12]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.hr_staff.full_name","service":"crm","table":"hr_staff","column":"full_name","type":"character varying(100)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"full_name\""],[45,"values","96% of values are known names"]],"shapes":[["Aa+ Aa+",98],["Aa+ A'Aa+",1]],"examples":{"token":"NAME_cca156e4","initial":"O••• N•••••••","redact":"••••••••","null":null}},{"id":"crm.hr_staff.hire_date","service":"crm","table":"hr_staff","column":"hire_date","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"hire_date\" matches no class"],[0,"context","The name says this is another kind of date"]]},{"id":"crm.hr_staff.job_title","service":"crm","table":"hr_staff","column":"job_title","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"job_title\" matches no class"]]},{"id":"crm.hr_staff.manager_emp_no","service":"crm","table":"hr_staff","column":"manager_emp_no","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"manager_emp_no\" matches no class"]]},{"id":"crm.hr_staff.ni_number","service":"crm","table":"hr_staff","column":"ni_number","type":"character varying(13)","verdict":"masked","class":"national_id","tier":"restricted","confidence":75,"total":75,"lines":[[0,"name","Column name \"ni_number\" matches no class"],[75,"values","92 of 92 values are valid UK NI numbers"]],"shapes":[["A+999999A",67],["A+ 99 99 99 A",32]],"examples":{"redact":"••••••••","last4":"•• •• •6 46 C","null":null}},{"id":"crm.hr_staff.passport_country","service":"crm","table":"hr_staff","column":"passport_country","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[15,"name","Column name \"passport_country\""],[0,"values","0 of 256 values look like passport numbers (few distinct values)"]]},{"id":"crm.hr_staff.passport_no","service":"crm","table":"hr_staff","column":"passport_no","type":"character varying(12)","verdict":"masked","class":"passport_number","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"passport_no\""],[20,"values","109 of 109 values look like passport numbers"]],"shapes":[["999999999",38],["99A+99999",26],["A9999999",18]],"examples":{"redact":"••••••••","last4":"•••••8045","null":null}},{"id":"crm.hr_staff.salary","service":"crm","table":"hr_staff","column":"salary","type":"numeric(10,2)","verdict":"masked","class":"financial_personal","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"salary\""],[20,"values","256 of 256 values are money amounts"]],"shapes":[["999999.99",59],["99999.99",40]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.hr_staff.ssn_verified","service":"crm","table":"hr_staff","column":"ssn_verified","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ssn_verified\" matches no class"],[0,"context","A boolean column can't hold national id"]]},{"id":"crm.hr_staff.term_date","service":"crm","table":"hr_staff","column":"term_date","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"term_date\" matches no class"]]},{"id":"crm.hr_staff.union_member","service":"crm","table":"hr_staff","column":"union_member","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"union_member\" matches no class"]]},{"id":"crm.hr_staff.work_email","service":"crm","table":"hr_staff","column":"work_email","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"work_email\""],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",30],["a+.a@a+.a+",16],["a+99@a+.a+",15]],"examples":{"token":"EMAIL_9b673e15","initial":"o••••••••••••••@••••••••••••••••••.com","redact":"••••••••","null":null}},{"id":"crm.klanten.aangemaakt_op","service":"crm","table":"klanten","column":"aangemaakt_op","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"aangemaakt_op\" matches no class"]]},{"id":"crm.klanten.achternaam","service":"crm","table":"klanten","column":"achternaam","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"achternaam\""],[45,"values","96% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_ffa26f6c","initial":"V••","redact":"••••••••","null":null}},{"id":"crm.klanten.bsn","service":"crm","table":"klanten","column":"bsn","type":"character(9)","verdict":"masked","class":"national_id","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"bsn\""],[45,"values","250 of 250 values are valid Dutch BSNs"]],"shapes":[["999999999",100]],"examples":{"redact":"••••••••","last4":"•••••3652","null":null}},{"id":"crm.klanten.emailadres","service":"crm","table":"klanten","column":"emailadres","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"emailadres\""],[75,"values","250 of 250 values are valid email addresses"]],"shapes":[["a+@a+.a+",32],["a+.a+@a+.a+",16],["a+_a+99@a+.a+",14]],"examples":{"token":"EMAIL_6423488b","initial":"h•••••••••@•••••.nl","redact":"••••••••","null":null}},{"id":"crm.klanten.geboortedatum","service":"crm","table":"klanten","column":"geboortedatum","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"geboortedatum\""],[45,"values","Values look like birth dates (67 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1985-••-••","redact":"••••••••","token":"DOB_c450b3bc","null":null}},{"id":"crm.klanten.geslacht","service":"crm","table":"klanten","column":"geslacht","type":"character(1)","verdict":"masked","class":"gender","tier":"quasi","confidence":80,"total":80,"lines":[[70,"name","Column name \"geslacht\""],[10,"values","250 of 250 values are gender values (few distinct values)"]],"shapes":[["A",100]],"examples":{"redact":"••••••••","token":"GENDER_ea07bdeb","null":null}},{"id":"crm.klanten.huisnummer","service":"crm","table":"klanten","column":"huisnummer","type":"character varying(10)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"huisnummer\""],[0,"values","0 of 250 values look like street addresses"]],"shapes":[["999",27],["99",21],["999A",11]],"examples":{"redact":"••••••••","token":"ADDR_19ca7cc4","null":null}},{"id":"crm.klanten.klant_id","service":"crm","table":"klanten","column":"klant_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"klant_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.klanten.postcode","service":"crm","table":"klanten","column":"postcode","type":"character(7)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":100,"total":100,"lines":[[70,"name","Column name \"postcode\""],[45,"values","250 of 250 values are Dutch postcodes"]],"shapes":[["9999 A+",100]],"examples":{"redact":"••••••••","token":"ZIP_74902aa7","null":null}},{"id":"crm.klanten.rekeningnummer","service":"crm","table":"klanten","column":"rekeningnummer","type":"character varying(34)","verdict":"masked","class":"bank_account","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"rekeningnummer\""],[75,"values","250 of 250 values are valid IBANs"]],"shapes":[["A+99A+9999999999",66],["A+99 A+ 9999 9999 99",33]],"examples":{"last4":"••••••••••••••4006","redact":"••••••••","null":null}},{"id":"crm.klanten.straatnaam","service":"crm","table":"klanten","column":"straatnaam","type":"character varying(80)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"straatnaam\""],[0,"values","0 of 250 values look like street addresses"]],"shapes":[["Aa+",100]],"examples":{"redact":"••••••••","token":"ADDR_1a47ad21","null":null}},{"id":"crm.klanten.telefoonnummer","service":"crm","table":"klanten","column":"telefoonnummer","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"telefoonnummer\""],[45,"values","250 of 250 values are formatted phone numbers"]],"shapes":[["99-99999999",28],["+99 9 99999999",27],["999-9999999",26]],"examples":{"token":"PHONE_1987e0e3","last4":"••-••••9691","redact":"••••••••","null":null}},{"id":"crm.klanten.tussenvoegsel","service":"crm","table":"klanten","column":"tussenvoegsel","type":"character varying(10)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"tussenvoegsel\""],[0,"values","0% of values are known names"]],"shapes":[["a+",78],["a+ a+",21]],"examples":{"token":"NAME_0c99d024","initial":"d•","redact":"••••••••","null":null}},{"id":"crm.klanten.voornaam","service":"crm","table":"klanten","column":"voornaam","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"voornaam\""],[45,"values","100% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_21742e02","initial":"H•••","redact":"••••••••","null":null}},{"id":"crm.klanten.woonplaats","service":"crm","table":"klanten","column":"woonplaats","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"woonplaats\" matches no class"]]},{"id":"crm.kunden.anrede","service":"crm","table":"kunden","column":"anrede","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"anrede\" matches no class"]]},{"id":"crm.kunden.email","service":"crm","table":"kunden","column":"email","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"email\""],[75,"values","247 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",25],["a+.a+@a+.a+",11],["a+_a+99@a+.a+",11]],"examples":{"token":"EMAIL_882db7d8","initial":"j••••••@•••.de","redact":"••••••••","null":null}},{"id":"crm.kunden.erstellt_am","service":"crm","table":"kunden","column":"erstellt_am","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"erstellt_am\" matches no class"]]},{"id":"crm.kunden.geburtsdatum","service":"crm","table":"kunden","column":"geburtsdatum","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"geburtsdatum\""],[45,"values","Values look like birth dates (67 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1975-••-••","redact":"••••••••","token":"DOB_7fe3294e","null":null}},{"id":"crm.kunden.hausnummer","service":"crm","table":"kunden","column":"hausnummer","type":"character varying(8)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"hausnummer\""],[0,"values","0 of 256 values look like street addresses"]],"shapes":[["99",32],["999",30],["99a",17]],"examples":{"redact":"••••••••","token":"ADDR_bedd22a1","null":null}},{"id":"crm.kunden.iban","service":"crm","table":"kunden","column":"iban","type":"character varying(34)","verdict":"masked","class":"bank_account","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"iban\""],[75,"values","192 of 192 values are valid IBANs"]],"shapes":[["A+99999999999999999999",63],["A+99 9999 9999 9999 9999 99",36]],"examples":{"last4":"••••••••••••••••••7008","redact":"••••••••","null":null}},{"id":"crm.kunden.konfession","service":"crm","table":"kunden","column":"konfession","type":"character varying(4)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"konfession\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes (few distinct values)"]],"shapes":[["a+",100]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.kunden.kunden_nr","service":"crm","table":"kunden","column":"kunden_nr","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"kunden_nr\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.kunden.kundengruppe","service":"crm","table":"kunden","column":"kundengruppe","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"kundengruppe\" matches no class"]]},{"id":"crm.kunden.nachname","service":"crm","table":"kunden","column":"nachname","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"nachname\""],[45,"values","100% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_416e8c46","initial":"S••••••••","redact":"••••••••","null":null}},{"id":"crm.kunden.ort","service":"crm","table":"kunden","column":"ort","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ort\" matches no class"]]},{"id":"crm.kunden.plz","service":"crm","table":"kunden","column":"plz","type":"character(5)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"plz\""],[20,"values","256 of 256 values look like postal codes"]],"shapes":[["99999",100]],"examples":{"redact":"••••••••","token":"ZIP_d6b82219","null":null}},{"id":"crm.kunden.steuer_id","service":"crm","table":"kunden","column":"steuer_id","type":"character(11)","verdict":"masked","class":"tax_id","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"steuer_id\""],[45,"values","206 of 206 values are valid German tax IDs"]],"shapes":[["99999999999",100]],"examples":{"redact":"••••••••","last4":"•••••••1564","null":null}},{"id":"crm.kunden.strasse","service":"crm","table":"kunden","column":"strasse","type":"character varying(80)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"strasse\""],[0,"values","0 of 256 values look like street addresses"]],"shapes":[["Aa+",96],["Aa+ Aa+",3]],"examples":{"redact":"••••••••","token":"ADDR_b07c0723","null":null}},{"id":"crm.kunden.telefon","service":"crm","table":"kunden","column":"telefon","type":"character varying(25)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"telefon\""],[35,"values","171 of 256 values are formatted phone numbers"]],"shapes":[["+999999999999",34],["+99 999 99999999",32],["999/9999999",23]],"examples":{"token":"PHONE_77c8a3d8","last4":"•••/•••1765","redact":"••••••••","null":null}},{"id":"crm.kunden.umsatz_ytd","service":"crm","table":"kunden","column":"umsatz_ytd","type":"numeric(10,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"umsatz_ytd\" matches no class"]]},{"id":"crm.kunden.vorname","service":"crm","table":"kunden","column":"vorname","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"vorname\""],[45,"values","96% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_99a51992","initial":"J••••","redact":"••••••••","null":null}},{"id":"crm.kunder.antal_ordrar","service":"crm","table":"kunder","column":"antal_ordrar","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"antal_ordrar\" matches no class"]]},{"id":"crm.kunder.efternamn","service":"crm","table":"kunder","column":"efternamn","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"efternamn\""],[45,"values","92% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_a7706962","initial":"S•••••••","redact":"••••••••","null":null}},{"id":"crm.kunder.epost","service":"crm","table":"kunder","column":"epost","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"epost\""],[75,"values","250 of 250 values are valid email addresses"]],"shapes":[["a+@a+.a+",30],["a+99@a+.a+",16],["a+.a+99@a+.a+",15]],"examples":{"token":"EMAIL_e17e8de4","initial":"b•••••••••••••••@•••••.com","redact":"••••••••","null":null}},{"id":"crm.kunder.fornamn","service":"crm","table":"kunder","column":"fornamn","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"fornamn\""],[45,"values","94% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_3f675dad","initial":"B••••","redact":"••••••••","null":null}},{"id":"crm.kunder.gatuadress","service":"crm","table":"kunder","column":"gatuadress","type":"character varying(80)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"gatuadress\""],[45,"values","250 of 250 values look like street addresses"]],"shapes":[["Aa+ 99",87],["Aa+ 9",12]],"examples":{"redact":"••••••••","token":"ADDR_8625f991","null":null}},{"id":"crm.kunder.kon","service":"crm","table":"kunder","column":"kon","type":"character varying(6)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"kon\" matches no class"]]},{"id":"crm.kunder.kund_id","service":"crm","table":"kunder","column":"kund_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"kund_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.kunder.lon","service":"crm","table":"kunder","column":"lon","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"lon\" matches no class"],[0,"context","An integer column can't hold precise location"]]},{"id":"crm.kunder.medlem_sedan","service":"crm","table":"kunder","column":"medlem_sedan","type":"date","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"medlem_sedan\" matches no class"],[45,"values","Values look like birth dates (19 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"2017-••-••","redact":"••••••••","token":"DOB_6e71a08d","null":null}},{"id":"crm.kunder.mobilnummer","service":"crm","table":"kunder","column":"mobilnummer","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"mobilnummer\""],[45,"values","250 of 250 values are formatted phone numbers"]],"shapes":[["+99999999999",36],["+99 99 999 99 99",32],["999-999 99 99",32]],"examples":{"token":"PHONE_1827db7b","last4":"+•••••••1236","redact":"••••••••","null":null}},{"id":"crm.kunder.ort","service":"crm","table":"kunder","column":"ort","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ort\" matches no class"]]},{"id":"crm.kunder.personnummer","service":"crm","table":"kunder","column":"personnummer","type":"character varying(13)","verdict":"masked","class":"national_id","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"personnummer\""],[45,"values","250 of 250 values are valid Swedish personal numbers"]],"shapes":[["99999999-9999",60],["999999-9999",39]],"examples":{"redact":"••••••••","last4":"••••••••-9723","null":null}},{"id":"crm.kunder.postnummer","service":"crm","table":"kunder","column":"postnummer","type":"character(6)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":70,"total":70,"lines":[[70,"name","Column name \"postnummer\""],[0,"values","0 of 250 values are US ZIP+4 codes"]],"shapes":[["999 99",100]],"examples":{"redact":"••••••••","token":"ZIP_fc9a35d7","null":null}},{"id":"crm.legacy_feed.attr_value","service":"crm","table":"legacy_feed","column":"attr_value","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":75,"total":75,"lines":[[0,"name","Column name \"attr_value\" matches no class"],[75,"values","229 of 229 values are valid email addresses"]],"shapes":[["a+@a+.a+",30],["a+.a@a+.a+",16],["a+99@a+.a+",14]],"examples":{"token":"EMAIL_e096d083","initial":"d•••••••••@•••.com","redact":"••••••••","null":null}},{"id":"crm.legacy_feed.data","service":"crm","table":"legacy_feed","column":"data","type":"text","verdict":"masked","class":"free_text_pii","tier":"free_text","confidence":75,"total":75,"lines":[[0,"name","Column name \"data\" matches no class"],[75,"text","Personal data inside the text in 95 rows"]],"shapes":[["{\"a+\": \"a+\", \"a\": 9, \"a+\": [\"a\"]}",27]],"examples":{"span":"Call Jane at •••••• or ••••••","redact":"••••••••","null":null}},{"id":"crm.legacy_feed.extra1","service":"crm","table":"legacy_feed","column":"extra1","type":"character varying(34)","verdict":"masked","class":"bank_account","tier":"restricted","confidence":75,"total":75,"lines":[[0,"name","Column name \"extra1\" matches no class"],[75,"values","28 of 28 values are valid IBANs"]],"shapes":[["A+99999999999999999999",57],["A+99A+99999999999999",42]],"examples":{"last4":"••••••••••••••••••2444","redact":"••••••••","null":null}},{"id":"crm.legacy_feed.field_1","service":"crm","table":"legacy_feed","column":"field_1","type":"character varying(80)","verdict":"review","class":"person_name","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"field_1\" matches no class"],[45,"values","97% of values are known names"]],"shapes":[["A+, A+",62],["A+, A+ A",35],["A'A+, A+ A",1]],"examples":{"token":"NAME_f88326df","initial":"C•••• D•••••","redact":"••••••••","null":null}},{"id":"crm.legacy_feed.field_12","service":"crm","table":"legacy_feed","column":"field_12","type":"character varying(20)","verdict":"review","class":"phone","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"field_12\" matches no class"],[45,"values","256 of 256 values are formatted phone numbers"]],"shapes":[["999.999.9999",25],["999-999-9999",20],["+9 999 999 9999",19]],"examples":{"token":"PHONE_3d6c9861","last4":"+• ••• ••• 2897","redact":"••••••••","null":null}},{"id":"crm.legacy_feed.field_3","service":"crm","table":"legacy_feed","column":"field_3","type":"character varying(10)","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"field_3\" matches no class"],[45,"values","Values look like birth dates (67 distinct years)"]],"shapes":[["99/99/9999",100]],"examples":{"year":"••/••/1942","redact":"••••••••","token":"DOB_aa37488a","null":null}},{"id":"crm.legacy_feed.field_7","service":"crm","table":"legacy_feed","column":"field_7","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"field_7\" matches no class"]]},{"id":"crm.legacy_feed.loaded_at","service":"crm","table":"legacy_feed","column":"loaded_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"loaded_at\" matches no class"]]},{"id":"crm.legacy_feed.misc","service":"crm","table":"legacy_feed","column":"misc","type":"character varying(120)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"misc\" matches no class"]]},{"id":"crm.legacy_feed.rec_id","service":"crm","table":"legacy_feed","column":"rec_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rec_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.legacy_feed.src_system","service":"crm","table":"legacy_feed","column":"src_system","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"src_system\" matches no class"]]},{"id":"crm.legacy_feed.value","service":"crm","table":"legacy_feed","column":"value","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"value\" matches no class"]]},{"id":"crm.loyalty_members.age","service":"crm","table":"loyalty_members","column":"age","type":"smallint","verdict":"masked","class":"age","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"age\" in a person table"],[20,"values","256 of 256 values are plausible ages"]],"shapes":[["99",100]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.loyalty_members.age_band","service":"crm","table":"loyalty_members","column":"age_band","type":"character varying(8)","verdict":"clear","class":null,"tier":null,"confidence":30,"total":30,"lines":[[70,"name","Column name \"age_band\" in a person table"],[0,"values","0 of 256 values are plausible ages (few distinct values)"]]},{"id":"crm.loyalty_members.birthday_mmdd","service":"crm","table":"loyalty_members","column":"birthday_mmdd","type":"character(5)","verdict":"clear","class":null,"tier":null,"confidence":30,"total":30,"lines":[[70,"name","Column name \"birthday_mmdd\""],[0,"values","Values don't look like birth dates"]]},{"id":"crm.loyalty_members.favorite_store_id","service":"crm","table":"loyalty_members","column":"favorite_store_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"favorite_store_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.loyalty_members.first_name","service":"crm","table":"loyalty_members","column":"first_name","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"first_name\""],[45,"values","88% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_a82ee22e","initial":"K•••••","redact":"••••••••","null":null}},{"id":"crm.loyalty_members.gender","service":"crm","table":"loyalty_members","column":"gender","type":"character varying(20)","verdict":"masked","class":"gender","tier":"quasi","confidence":80,"total":80,"lines":[[70,"name","Column name \"gender\""],[10,"values","256 of 256 values are gender values (few distinct values)"]],"shapes":[["a+",95],["a+ a+ a+ a+",3],["a+-a+",1]],"examples":{"redact":"••••••••","token":"GENDER_eb581206","null":null}},{"id":"crm.loyalty_members.home_postal_code","service":"crm","table":"loyalty_members","column":"home_postal_code","type":"character varying(10)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"home_postal_code\""],[20,"values","211 of 256 values look like postal codes"]],"shapes":[["99999",82],["99999-9999",17]],"examples":{"redact":"••••••••","token":"ZIP_623ed338","null":null}},{"id":"crm.loyalty_members.loyalty_card_no","service":"crm","table":"loyalty_members","column":"loyalty_card_no","type":"character(16)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"loyalty_card_no\" matches no class"],[0,"context","\"loyalty card number\" rules out card number"]]},{"id":"crm.loyalty_members.member_id","service":"crm","table":"loyalty_members","column":"member_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"member_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.loyalty_members.member_since","service":"crm","table":"loyalty_members","column":"member_since","type":"date","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"member_since\" matches no class"],[45,"values","Values look like birth dates (15 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"2016-••-••","redact":"••••••••","token":"DOB_c5e889bb","null":null}},{"id":"crm.loyalty_members.points_balance","service":"crm","table":"loyalty_members","column":"points_balance","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"points_balance\" matches no class"]]},{"id":"crm.loyalty_members.sms_opt_in","service":"crm","table":"loyalty_members","column":"sms_opt_in","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sms_opt_in\" matches no class"]]},{"id":"crm.loyalty_members.tier","service":"crm","table":"loyalty_members","column":"tier","type":"character varying(8)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"tier\" matches no class"]]},{"id":"crm.newsletter_signups.birth_date","service":"crm","table":"newsletter_signups","column":"birth_date","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"birth_date\""],[45,"values","Values look like birth dates (21 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1981-••-••","redact":"••••••••","token":"DOB_c76dd6c4","null":null}},{"id":"crm.newsletter_signups.consent_ts","service":"crm","table":"newsletter_signups","column":"consent_ts","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"consent_ts\" matches no class"]]},{"id":"crm.newsletter_signups.double_opt_in","service":"crm","table":"newsletter_signups","column":"double_opt_in","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"double_opt_in\" matches no class"]]},{"id":"crm.newsletter_signups.email","service":"crm","table":"newsletter_signups","column":"email","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"email\""],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",26],["a+_a+99@a+.a+",16],["a+99@a+.a+",14]],"examples":{"token":"EMAIL_7588d901","initial":"b••••••••••••@••••••.com","redact":"••••••••","null":null}},{"id":"crm.newsletter_signups.fax","service":"crm","table":"newsletter_signups","column":"fax","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"fax\""],[0,"context","Only 4 non-null values sampled"],[0,"values","4 of 4 values are formatted phone numbers (few distinct values)"]],"shapes":[["999-999-9999",100]],"examples":{"token":"PHONE_c767ffce","last4":"+• •••-•••-0132","redact":"••••••••","null":null}},{"id":"crm.newsletter_signups.middle_name","service":"crm","table":"newsletter_signups","column":"middle_name","type":"character varying(40)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"middle_name\""],[0,"context","Only 0 non-null values sampled"]],"examples":{"token":"NAME_c0b3ed2e","initial":"J••• D••","redact":"••••••••","null":null}},{"id":"crm.newsletter_signups.phone","service":"crm","table":"newsletter_signups","column":"phone","type":"character varying(20)","verdict":"masked","class":"phone","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"phone\""],[45,"values","11 of 11 values are formatted phone numbers"]],"shapes":[["999-999-9999",36],["(999) 999-9999",18],["+9 999 999 9999",18]],"examples":{"token":"PHONE_3202986d","last4":"(•••) •••-6646","redact":"••••••••","null":null}},{"id":"crm.newsletter_signups.referrer_name","service":"crm","table":"newsletter_signups","column":"referrer_name","type":"character varying(80)","verdict":"review","class":"person_name","tier":"identity","confidence":60,"total":60,"lines":[[15,"name","Column name \"referrer_name\""],[45,"values","100% of values are known names"]],"shapes":[["Aa+ Aa+",100]],"examples":{"token":"NAME_20eaa6ab","initial":"D••••• W•••••••","redact":"••••••••","null":null}},{"id":"crm.newsletter_signups.signup_id","service":"crm","table":"newsletter_signups","column":"signup_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"signup_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.newsletter_signups.signup_ip","service":"crm","table":"newsletter_signups","column":"signup_ip","type":"character varying(45)","verdict":"masked","class":"ip_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"signup_ip\""],[45,"values","103 of 128 values are IPv4 addresses"]],"shapes":[["999.999.999.999",10],["999.999.99.999",7],["99.999.99.999",5]],"examples":{"token":"IP_d8c822ae","redact":"••••••••","null":null}},{"id":"crm.newsletter_signups.utm_campaign","service":"crm","table":"newsletter_signups","column":"utm_campaign","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"utm_campaign\" matches no class"]]},{"id":"crm.newsletter_signups.utm_source","service":"crm","table":"newsletter_signups","column":"utm_source","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"utm_source\" matches no class"]]},{"id":"crm.orders_ledger.card_last4","service":"crm","table":"orders_ledger","column":"card_last4","type":"character(4)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"card_last4\" matches no class"]]},{"id":"crm.orders_ledger.card_type","service":"crm","table":"orders_ledger","column":"card_type","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"card_type\" matches no class"],[0,"context","\"card type\" rules out card number"]]},{"id":"crm.orders_ledger.currency","service":"crm","table":"orders_ledger","column":"currency","type":"character(3)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"currency\" matches no class"]]},{"id":"crm.orders_ledger.customer_id","service":"crm","table":"orders_ledger","column":"customer_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customer_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.orders_ledger.customer_po","service":"crm","table":"orders_ledger","column":"customer_po","type":"character varying(13)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customer_po\" matches no class"]]},{"id":"crm.orders_ledger.order_id","service":"crm","table":"orders_ledger","column":"order_id","type":"bigint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"order_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.orders_ledger.order_number","service":"crm","table":"orders_ledger","column":"order_number","type":"character varying(16)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"order_number\" matches no class"]]},{"id":"crm.orders_ledger.order_total","service":"crm","table":"orders_ledger","column":"order_total","type":"numeric(10,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"order_total\" matches no class"]]},{"id":"crm.orders_ledger.part_no","service":"crm","table":"orders_ledger","column":"part_no","type":"character varying(11)","verdict":"masked","class":"national_id","tier":"restricted","confidence":75,"total":75,"lines":[[0,"name","Column name \"part_no\" matches no class"],[75,"values","256 of 256 values are valid US SSNs"]],"shapes":[["999-99-9999",100]],"examples":{"redact":"••••••••","last4":"•••-••-9997","null":null}},{"id":"crm.orders_ledger.passenger_count","service":"crm","table":"orders_ledger","column":"passenger_count","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"passenger_count\" matches no class"]]},{"id":"crm.orders_ledger.placed_at","service":"crm","table":"orders_ledger","column":"placed_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"placed_at\" matches no class"]]},{"id":"crm.orders_ledger.product_code","service":"crm","table":"orders_ledger","column":"product_code","type":"character varying(5)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"product_code\" matches no class"]]},{"id":"crm.orders_ledger.ship_city","service":"crm","table":"orders_ledger","column":"ship_city","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ship_city\" matches no class"]]},{"id":"crm.orders_ledger.ship_country","service":"crm","table":"orders_ledger","column":"ship_country","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ship_country\" matches no class"]]},{"id":"crm.orders_ledger.ship_state","service":"crm","table":"orders_ledger","column":"ship_state","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ship_state\" matches no class"]]},{"id":"crm.orders_ledger.statement_date","service":"crm","table":"orders_ledger","column":"statement_date","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"statement_date\" matches no class"]]},{"id":"crm.orders_ledger.tracking_no","service":"crm","table":"orders_ledger","column":"tracking_no","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"tracking_no\" matches no class"]]},{"id":"crm.patient_encounters.allergies","service":"crm","table":"patient_encounters","column":"allergies","type":"character varying(60)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"allergies\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes (few distinct values)"]],"shapes":[["A+",63],["a+",29],["a+ a+",6]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.patient_encounters.attending_physician","service":"crm","table":"patient_encounters","column":"attending_physician","type":"character varying(60)","verdict":"review","class":"person_name","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"attending_physician\" matches no class"],[45,"values","80% of values are known names"]],"shapes":[["Aa+. A. Aa+",96],["Aa+. A. A'Aa+",3]],"examples":{"token":"NAME_fff277da","initial":"D•• P• A•••••••","redact":"••••••••","null":null}},{"id":"crm.patient_encounters.blood_type","service":"crm","table":"patient_encounters","column":"blood_type","type":"character varying(3)","verdict":"masked","class":"special_category","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"blood_type\""],[20,"values","256 of 256 values are blood types"]],"shapes":[["A+",36],["A-",36],["A+-",13]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.patient_encounters.bmi","service":"crm","table":"patient_encounters","column":"bmi","type":"numeric(4,1)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"bmi\" matches no class"]]},{"id":"crm.patient_encounters.clinic_city","service":"crm","table":"patient_encounters","column":"clinic_city","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"clinic_city\" matches no class"]]},{"id":"crm.patient_encounters.clinical_notes","service":"crm","table":"patient_encounters","column":"clinical_notes","type":"text","verdict":"review","class":"free_text_pii","tier":"free_text","confidence":45,"total":45,"lines":[[0,"name","Column name \"clinical_notes\" matches no class"],[45,"text","Possible personal data inside the text in 22 rows"]],"examples":{"span":"Call Jane at •••••• or ••••••","redact":"••••••••","null":null}},{"id":"crm.patient_encounters.copay_amount","service":"crm","table":"patient_encounters","column":"copay_amount","type":"numeric(7,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"copay_amount\" matches no class"]]},{"id":"crm.patient_encounters.diagnosis","service":"crm","table":"patient_encounters","column":"diagnosis","type":"character varying(120)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"diagnosis\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes"]],"shapes":[["Aa+ a+ a+",17],["A+ a+",9],["Aa+-a+ a+ a+",9]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.patient_encounters.encounter_date","service":"crm","table":"patient_encounters","column":"encounter_date","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"encounter_date\" matches no class"]]},{"id":"crm.patient_encounters.encounter_id","service":"crm","table":"patient_encounters","column":"encounter_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"encounter_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.patient_encounters.icd10_code","service":"crm","table":"patient_encounters","column":"icd10_code","type":"character varying(8)","verdict":"masked","class":"special_category","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"icd10_code\""],[20,"values","256 of 256 values are ICD-10 diagnosis codes"]],"shapes":[["A99.9",51],["A99.99",16],["A99.999",16]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.patient_encounters.medication","service":"crm","table":"patient_encounters","column":"medication","type":"character varying(60)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"medication\""],[0,"values","0 of 232 values are ICD-10 diagnosis codes"]],"shapes":[["Aa+ 99 a+",56],["Aa+ a+",17],["Aa+ 999 a+",15]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.patient_encounters.patient_id","service":"crm","table":"patient_encounters","column":"patient_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"patient_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.patient_encounters.religion","service":"crm","table":"patient_encounters","column":"religion","type":"character varying(20)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"religion\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes (few distinct values)"]],"shapes":[["Aa+",100]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.payment_methods.billing_postcode","service":"crm","table":"payment_methods","column":"billing_postcode","type":"character varying(9)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":100,"total":100,"lines":[[70,"name","Column name \"billing_postcode\""],[45,"values","220 of 256 values are UK postcodes"]],"shapes":[["A+99 9A+",21],["A+9 9A+",19],["A99 9A+",16]],"examples":{"redact":"••••••••","token":"ZIP_e7794bcc","null":null}},{"id":"crm.payment_methods.bin6","service":"crm","table":"payment_methods","column":"bin6","type":"character(6)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"bin6\" matches no class"]]},{"id":"crm.payment_methods.brand","service":"crm","table":"payment_methods","column":"brand","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"brand\" matches no class"]]},{"id":"crm.payment_methods.cardholder_name","service":"crm","table":"payment_methods","column":"cardholder_name","type":"character varying(80)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"cardholder_name\""],[45,"values","97% of values are known names"]],"shapes":[["A+ A+",51],["Aa+ Aa+",47],["A+ A'A+",0]],"examples":{"token":"NAME_5c068c18","initial":"S••• H•••","redact":"••••••••","null":null}},{"id":"crm.payment_methods.created_at","service":"crm","table":"payment_methods","column":"created_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"created_at\" matches no class"]]},{"id":"crm.payment_methods.customer_id","service":"crm","table":"payment_methods","column":"customer_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customer_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.payment_methods.cvc","service":"crm","table":"payment_methods","column":"cvc","type":"character varying(4)","verdict":"masked","class":"card_security","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"cvc\""],[20,"values","176 of 176 values look like card security codes"]],"shapes":[["999",85],["9999",14]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.payment_methods.exp_month","service":"crm","table":"payment_methods","column":"exp_month","type":"smallint","verdict":"masked","class":"card_expiry","tier":"quasi","confidence":70,"total":70,"lines":[[70,"name","Column name \"exp_month\""],[0,"values","0 of 256 values look like card expiry dates"]],"shapes":[["9",75],["99",24]],"examples":{"redact":"••••••••","year":"••••••••","null":null}},{"id":"crm.payment_methods.exp_year","service":"crm","table":"payment_methods","column":"exp_year","type":"smallint","verdict":"masked","class":"card_expiry","tier":"quasi","confidence":70,"total":70,"lines":[[70,"name","Column name \"exp_year\""],[0,"values","0 of 256 values look like card expiry dates"]],"shapes":[["9999",100]],"examples":{"redact":"••••••••","year":"2026","null":null}},{"id":"crm.payment_methods.fingerprint","service":"crm","table":"payment_methods","column":"fingerprint","type":"character varying(16)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"fingerprint\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes"]]},{"id":"crm.payment_methods.is_default","service":"crm","table":"payment_methods","column":"is_default","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"is_default\" matches no class"]]},{"id":"crm.payment_methods.pan","service":"crm","table":"payment_methods","column":"pan","type":"character varying(19)","verdict":"masked","class":"card_number","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"pan\""],[75,"values","235 of 256 values pass the card checksum"]],"shapes":[["9999999999999999",51],["9999 9999 9999 9999",24],["9999-9999-9999-9999",10]],"examples":{"last4":"••••••••••••9499","redact":"••••••••","null":null}},{"id":"crm.payment_methods.pan_masked","service":"crm","table":"payment_methods","column":"pan_masked","type":"character varying(19)","verdict":"clear","class":null,"tier":null,"confidence":20,"total":20,"lines":[[0,"name","Column name \"pan_masked\""],[20,"values","256 of 256 values are already-masked card numbers"]]},{"id":"crm.payment_methods.pm_id","service":"crm","table":"payment_methods","column":"pm_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"pm_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.payment_methods.processor_pm_id","service":"crm","table":"payment_methods","column":"processor_pm_id","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"processor_pm_id\" matches no class"]]},{"id":"crm.products_catalog.description","service":"crm","table":"products_catalog","column":"description","type":"text","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"description\" matches no class"]]},{"id":"crm.products_catalog.ean13","service":"crm","table":"products_catalog","column":"ean13","type":"character(13)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ean13\" matches no class"]]},{"id":"crm.products_catalog.genre","service":"crm","table":"products_catalog","column":"genre","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"genre\" matches no class"]]},{"id":"crm.products_catalog.label_name","service":"crm","table":"products_catalog","column":"label_name","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"label_name\""],[0,"values","0% of values are known names"]]},{"id":"crm.products_catalog.list_price","service":"crm","table":"products_catalog","column":"list_price","type":"numeric(7,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"list_price\" matches no class"]]},{"id":"crm.products_catalog.name","service":"crm","table":"products_catalog","column":"name","type":"character varying(80)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"name\""],[0,"values","0% of values are known names"]]},{"id":"crm.products_catalog.nom","service":"crm","table":"products_catalog","column":"nom","type":"character varying(80)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"nom\""],[0,"values","0% of values are known names"]]},{"id":"crm.products_catalog.original_release_date","service":"crm","table":"products_catalog","column":"original_release_date","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"original_release_date\" matches no class"],[0,"context","The name says this is another kind of date"]]},{"id":"crm.products_catalog.product_code","service":"crm","table":"products_catalog","column":"product_code","type":"character(5)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"product_code\" matches no class"]]},{"id":"crm.products_catalog.product_id","service":"crm","table":"products_catalog","column":"product_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"product_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.products_catalog.reissue_date","service":"crm","table":"products_catalog","column":"reissue_date","type":"date","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"reissue_date\" matches no class"],[45,"values","Values look like birth dates (17 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"2015-••-••","redact":"••••••••","token":"DOB_7ca9d653","null":null}},{"id":"crm.products_catalog.sku","service":"crm","table":"products_catalog","column":"sku","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sku\" matches no class"]]},{"id":"crm.products_catalog.upc","service":"crm","table":"products_catalog","column":"upc","type":"character(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"upc\" matches no class"]]},{"id":"crm.products_catalog.weight_g","service":"crm","table":"products_catalog","column":"weight_g","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"weight_g\" matches no class"]]},{"id":"crm.service_appointments.access_notes","service":"crm","table":"service_appointments","column":"access_notes","type":"text","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"access_notes\" matches no class"]]},{"id":"crm.service_appointments.appt_id","service":"crm","table":"service_appointments","column":"appt_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"appt_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.service_appointments.customer_id","service":"crm","table":"service_appointments","column":"customer_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customer_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.service_appointments.duration_min","service":"crm","table":"service_appointments","column":"duration_min","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"duration_min\" matches no class"]]},{"id":"crm.service_appointments.gate_code","service":"crm","table":"service_appointments","column":"gate_code","type":"character varying(8)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"gate_code\" matches no class"]]},{"id":"crm.service_appointments.scheduled_for","service":"crm","table":"service_appointments","column":"scheduled_for","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"scheduled_for\" matches no class"]]},{"id":"crm.service_appointments.service_type","service":"crm","table":"service_appointments","column":"service_type","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"service_type\" matches no class"]]},{"id":"crm.service_appointments.site_lat","service":"crm","table":"service_appointments","column":"site_lat","type":"numeric(9,6)","verdict":"review","class":"geo_precise","tier":"identity","confidence":60,"total":60,"lines":[[15,"name","Column name \"site_lat\""],[45,"values","256 of 256 values are precise coordinates"]],"shapes":[["99.999999",100]],"examples":{"redact":"••••••••","token":"GEO_14ac7d81","null":null}},{"id":"crm.service_appointments.site_lng","service":"crm","table":"service_appointments","column":"site_lng","type":"numeric(9,6)","verdict":"review","class":"geo_precise","tier":"identity","confidence":60,"total":60,"lines":[[15,"name","Column name \"site_lng\""],[45,"values","256 of 256 values are precise coordinates"]],"shapes":[["-99.999999",60],["-999.999999",39]],"examples":{"redact":"••••••••","token":"GEO_16cd7ba8","null":null}},{"id":"crm.service_appointments.status","service":"crm","table":"service_appointments","column":"status","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"status\" matches no class"]]},{"id":"crm.service_appointments.technician_emp_no","service":"crm","table":"service_appointments","column":"technician_emp_no","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"technician_emp_no\" matches no class"]]},{"id":"crm.staging_import.col1","service":"crm","table":"staging_import","column":"col1","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col1\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.staging_import.col10","service":"crm","table":"staging_import","column":"col10","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col10\" matches no class"]]},{"id":"crm.staging_import.col11","service":"crm","table":"staging_import","column":"col11","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col11\" matches no class"]]},{"id":"crm.staging_import.col12","service":"crm","table":"staging_import","column":"col12","type":"character varying(255)","verdict":"masked","class":"bank_account","tier":"restricted","confidence":75,"total":75,"lines":[[0,"name","Column name \"col12\" matches no class"],[75,"values","256 of 256 values are valid IBANs"]],"shapes":[["A+99A+99999999999999",15],["A+9999999999999999999999999",11],["A+99 9999 9999 9999 9999 99",11]],"examples":{"last4":"••••••••••••••1859","redact":"••••••••","null":null}},{"id":"crm.staging_import.col13","service":"crm","table":"staging_import","column":"col13","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col13\" matches no class"]]},{"id":"crm.staging_import.col14","service":"crm","table":"staging_import","column":"col14","type":"character varying(255)","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"col14\" matches no class"],[45,"values","Values look like birth dates (66 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1989-••-••","redact":"••••••••","token":"DOB_415b6e9d","null":null}},{"id":"crm.staging_import.col15","service":"crm","table":"staging_import","column":"col15","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col15\" matches no class"]]},{"id":"crm.staging_import.col16","service":"crm","table":"staging_import","column":"col16","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col16\" matches no class"]]},{"id":"crm.staging_import.col2","service":"crm","table":"staging_import","column":"col2","type":"character varying(255)","verdict":"masked","class":"email","tier":"identity","confidence":75,"total":75,"lines":[[0,"name","Column name \"col2\" matches no class"],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",31],["a+99@a+.a+",16],["a+.a+@a+.a+",12]],"examples":{"token":"EMAIL_1da6882b","initial":"l••••••••••••@•••••••.com","redact":"••••••••","null":null}},{"id":"crm.staging_import.col3","service":"crm","table":"staging_import","column":"col3","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col3\" matches no class"]]},{"id":"crm.staging_import.col4","service":"crm","table":"staging_import","column":"col4","type":"character varying(255)","verdict":"review","class":"phone","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"col4\" matches no class"],[45,"values","256 of 256 values are formatted phone numbers"]],"shapes":[["+99999999999",100]],"examples":{"token":"PHONE_a2f06870","last4":"+•••••••2542","redact":"••••••••","null":null}},{"id":"crm.staging_import.col5","service":"crm","table":"staging_import","column":"col5","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col5\" matches no class"]]},{"id":"crm.staging_import.col6","service":"crm","table":"staging_import","column":"col6","type":"character varying(255)","verdict":"masked","class":"card_number","tier":"restricted","confidence":75,"total":75,"lines":[[0,"name","Column name \"col6\" matches no class"],[75,"values","256 of 256 values pass the card checksum"]],"shapes":[["9999999999999999",80],["999999999999999",19]],"examples":{"last4":"••••••••••••6410","redact":"••••••••","null":null}},{"id":"crm.staging_import.col7","service":"crm","table":"staging_import","column":"col7","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col7\" matches no class"]]},{"id":"crm.staging_import.col8","service":"crm","table":"staging_import","column":"col8","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"col8\" matches no class"]]},{"id":"crm.staging_import.col9","service":"crm","table":"staging_import","column":"col9","type":"character varying(255)","verdict":"review","class":"person_name","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"col9\" matches no class"],[45,"values","97% of values are known names"]],"shapes":[["Aa+ Aa+",98],["Aa+ A'Aa+",1]],"examples":{"token":"NAME_d6b5eb12","initial":"L••• H••••••••","redact":"••••••••","null":null}},{"id":"crm.support_tickets.agent_notes","service":"crm","table":"support_tickets","column":"agent_notes","type":"text","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"agent_notes\" matches no class"]]},{"id":"crm.support_tickets.body","service":"crm","table":"support_tickets","column":"body","type":"text","verdict":"masked","class":"free_text_pii","tier":"free_text","confidence":75,"total":75,"lines":[[0,"name","Column name \"body\" matches no class"],[75,"text","Personal data inside the text in 14 rows"]],"shapes":[["Aa+ a+ a+ a+ a+ a+ a+ a+ a+ a+ a+.",13],["Aa+ a+ a+ a+ a+ a+, a+ a+ a+ a+ a+.",8],["Aa+ A a+ a+ a+ a+ a+ a+ a+ a+ a+?",8]],"examples":{"span":"Please update my email to ••••••, the old address no longer works.","redact":"••••••••","null":null}},{"id":"crm.support_tickets.channel","service":"crm","table":"support_tickets","column":"channel","type":"character varying(8)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"channel\" matches no class"]]},{"id":"crm.support_tickets.created_at","service":"crm","table":"support_tickets","column":"created_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"created_at\" matches no class"]]},{"id":"crm.support_tickets.csat_score","service":"crm","table":"support_tickets","column":"csat_score","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"csat_score\" matches no class"]]},{"id":"crm.support_tickets.customer_id","service":"crm","table":"support_tickets","column":"customer_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customer_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.support_tickets.priority","service":"crm","table":"support_tickets","column":"priority","type":"character varying(8)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"priority\" matches no class"]]},{"id":"crm.support_tickets.status","service":"crm","table":"support_tickets","column":"status","type":"character varying(10)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"status\" matches no class"]]},{"id":"crm.support_tickets.subject","service":"crm","table":"support_tickets","column":"subject","type":"character varying(120)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"subject\" matches no class"]]},{"id":"crm.support_tickets.ticket_id","service":"crm","table":"support_tickets","column":"ticket_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ticket_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.support_tickets.updated_at","service":"crm","table":"support_tickets","column":"updated_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"updated_at\" matches no class"]]},{"id":"crm.survey-responses.age-band","service":"crm","table":"survey-responses","column":"age-band","type":"character varying(8)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[15,"name","Column name \"age-band\""],[0,"values","0 of 256 values are plausible ages (few distinct values)"]]},{"id":"crm.survey-responses.e-mail","service":"crm","table":"survey-responses","column":"e-mail","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"e-mail\""],[75,"values","176 of 176 values are valid email addresses"]],"shapes":[["a+@a+.a+",28],["a+.a+99@a+.a+",16],["a+.a+@a+.a+",14]],"examples":{"token":"EMAIL_314a5fcc","initial":"s••••••••••@•••••••.net","redact":"••••••••","null":null}},{"id":"crm.survey-responses.first-name","service":"crm","table":"survey-responses","column":"first-name","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"first-name\""],[45,"values","87% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_e60b5973","initial":"L•••","redact":"••••••••","null":null}},{"id":"crm.survey-responses.free-comment","service":"crm","table":"survey-responses","column":"free-comment","type":"text","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"free-comment\" matches no class"]]},{"id":"crm.survey-responses.household-income","service":"crm","table":"survey-responses","column":"household-income","type":"character varying(16)","verdict":"clear","class":null,"tier":null,"confidence":30,"total":30,"lines":[[70,"name","Column name \"household-income\""],[0,"values","0 of 256 values are money amounts (few distinct values)"]]},{"id":"crm.survey-responses.q1-score","service":"crm","table":"survey-responses","column":"q1-score","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"q1-score\" matches no class"]]},{"id":"crm.survey-responses.q2-score","service":"crm","table":"survey-responses","column":"q2-score","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"q2-score\" matches no class"]]},{"id":"crm.survey-responses.response-id","service":"crm","table":"survey-responses","column":"response-id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"response-id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.survey-responses.sexual-orientation","service":"crm","table":"survey-responses","column":"sexual-orientation","type":"character varying(24)","verdict":"masked","class":"special_category","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"sexual-orientation\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes (few distinct values)"]],"shapes":[["a+",94],["a+ a+ a+ a+",5]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.survey-responses.submitted-at","service":"crm","table":"survey-responses","column":"submitted-at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"submitted-at\" matches no class"]]},{"id":"crm.survey-responses.would-recommend","service":"crm","table":"survey-responses","column":"would-recommend","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"would-recommend\" matches no class"]]},{"id":"crm.survey-responses.zip-code","service":"crm","table":"survey-responses","column":"zip-code","type":"character varying(10)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":80,"total":80,"lines":[[70,"name","Column name \"zip-code\""],[10,"values","201 of 256 values look like postal codes"]],"shapes":[["99999",78],["99999-9999",21]],"examples":{"redact":"••••••••","token":"ZIP_4a679379","null":null}},{"id":"crm.tbl_cust_master.acct_no","service":"crm","table":"tbl_cust_master","column":"acct_no","type":"character varying(17)","verdict":"clear","class":null,"tier":null,"confidence":35,"total":35,"lines":[[15,"name","Column name \"acct_no\""],[20,"values","256 of 256 values look like account numbers"]]},{"id":"crm.tbl_cust_master.addr1","service":"crm","table":"tbl_cust_master","column":"addr1","type":"character varying(120)","verdict":"masked","class":"street_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"addr1\""],[45,"values","256 of 256 values look like street addresses"]],"shapes":[["9999 Aa+ Aa+",86],["9999 9a+ Aa+",6],["999 Aa+ Aa+",5]],"examples":{"redact":"••••••••","token":"ADDR_6f492a0c","null":null}},{"id":"crm.tbl_cust_master.addr2","service":"crm","table":"tbl_cust_master","column":"addr2","type":"character varying(60)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"addr2\""],[0,"values","0 of 79 values look like street addresses"]],"shapes":[["Aa+ 99A",44],["Aa+ 99",17],["Aa+ 9A",16]],"examples":{"redact":"••••••••","token":"ADDR_fa978b15","null":null}},{"id":"crm.tbl_cust_master.cc_exp","service":"crm","table":"tbl_cust_master","column":"cc_exp","type":"character varying(5)","verdict":"masked","class":"card_expiry","tier":"quasi","confidence":100,"total":100,"lines":[[70,"name","Column name \"cc_exp\""],[45,"values","256 of 256 values look like card expiry dates"]],"shapes":[["99/99",100]],"examples":{"redact":"••••••••","year":"••/26","null":null}},{"id":"crm.tbl_cust_master.cc_num","service":"crm","table":"tbl_cust_master","column":"cc_num","type":"character varying(19)","verdict":"masked","class":"card_number","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"cc_num\""],[75,"values","246 of 256 values pass the card checksum"]],"shapes":[["9999999999999999",49],["9999 9999 9999 9999",28],["999999999999999",8]],"examples":{"last4":"••••-••••-••••-7768","redact":"••••••••","null":null}},{"id":"crm.tbl_cust_master.city","service":"crm","table":"tbl_cust_master","column":"city","type":"character varying(60)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"city\" matches no class"]]},{"id":"crm.tbl_cust_master.credit_limit","service":"crm","table":"tbl_cust_master","column":"credit_limit","type":"numeric(10,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"credit_limit\" matches no class"]]},{"id":"crm.tbl_cust_master.cust_fname","service":"crm","table":"tbl_cust_master","column":"cust_fname","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"cust_fname\""],[45,"values","82% of values are known names"]],"shapes":[["Aa+",100]],"examples":{"token":"NAME_0c62b5bb","initial":"H••••••","redact":"••••••••","null":null}},{"id":"crm.tbl_cust_master.cust_id","service":"crm","table":"tbl_cust_master","column":"cust_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"cust_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.tbl_cust_master.cust_lname","service":"crm","table":"tbl_cust_master","column":"cust_lname","type":"character varying(60)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"cust_lname\""],[35,"values","77% of values are known names"]],"shapes":[["Aa+",98],["A'Aa+",1]],"examples":{"token":"NAME_0873a9a2","initial":"K•••","redact":"••••••••","null":null}},{"id":"crm.tbl_cust_master.cust_since","service":"crm","table":"tbl_cust_master","column":"cust_since","type":"date","verdict":"review","class":"date_of_birth","tier":"identity","confidence":45,"total":45,"lines":[[0,"name","Column name \"cust_since\" matches no class"],[45,"values","Values look like birth dates (22 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"2011-••-••","redact":"••••••••","token":"DOB_cf32b519","null":null}},{"id":"crm.tbl_cust_master.cvv","service":"crm","table":"tbl_cust_master","column":"cvv","type":"character varying(4)","verdict":"masked","class":"card_security","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"cvv\""],[20,"values","256 of 256 values look like card security codes"]],"shapes":[["999",84],["9999",15]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.tbl_cust_master.dl_num","service":"crm","table":"tbl_cust_master","column":"dl_num","type":"character varying(20)","verdict":"masked","class":"drivers_license","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"dl_num\""],[20,"values","182 of 193 values look like driver's license numbers"]],"shapes":[["A99999999",60],["A9999999",9],["A999-9999-9999",8]],"examples":{"redact":"••••••••","last4":"•••••4512","null":null}},{"id":"crm.tbl_cust_master.dl_state","service":"crm","table":"tbl_cust_master","column":"dl_state","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"dl_state\" matches no class"],[0,"context","\"dl state\" rules out driver's license"]]},{"id":"crm.tbl_cust_master.dob","service":"crm","table":"tbl_cust_master","column":"dob","type":"date","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"dob\""],[45,"values","Values look like birth dates (65 distinct years)"]],"shapes":[["9999-99-99",100]],"examples":{"year":"1995-••-••","redact":"••••••••","token":"DOB_58887170","null":null}},{"id":"crm.tbl_cust_master.email_addr","service":"crm","table":"tbl_cust_master","column":"email_addr","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"email_addr\""],[0,"context","\"email address\" rules out street address"],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",29],["a+.a+99@a+.a+",17],["a+99@a+.a+",13]],"examples":{"token":"EMAIL_a4c56923","initial":"h••••••••••@•••••••.com","redact":"••••••••","null":null}},{"id":"crm.tbl_cust_master.gender_cd","service":"crm","table":"tbl_cust_master","column":"gender_cd","type":"character(1)","verdict":"masked","class":"gender","tier":"quasi","confidence":80,"total":80,"lines":[[70,"name","Column name \"gender_cd\""],[10,"values","256 of 256 values are gender values (few distinct values)"]],"shapes":[["A",100]],"examples":{"redact":"••••••••","token":"GENDER_ea07bdeb","null":null}},{"id":"crm.tbl_cust_master.mi","service":"crm","table":"tbl_cust_master","column":"mi","type":"character(1)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"mi\" matches no class"]]},{"id":"crm.tbl_cust_master.msisdn","service":"crm","table":"tbl_cust_master","column":"msisdn","type":"character varying(16)","verdict":"masked","class":"phone","tier":"identity","confidence":90,"total":90,"lines":[[70,"name","Column name \"msisdn\""],[20,"values","256 of 256 values are phone-length digit strings"]],"shapes":[["99999999999",85],["999999999999",14]],"examples":{"token":"PHONE_35cead95","last4":"•••••••0818","redact":"••••••••","null":null}},{"id":"crm.tbl_cust_master.row_ver","service":"crm","table":"tbl_cust_master","column":"row_ver","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"row_ver\" matches no class"]]},{"id":"crm.tbl_cust_master.ssn","service":"crm","table":"tbl_cust_master","column":"ssn","type":"character varying(11)","verdict":"masked","class":"national_id","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"ssn\""],[75,"values","256 of 256 values are valid US SSNs"]],"shapes":[["999-99-9999",100]],"examples":{"redact":"••••••••","last4":"•••-••-1891","null":null}},{"id":"crm.tbl_cust_master.st","service":"crm","table":"tbl_cust_master","column":"st","type":"character(2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"st\" matches no class"]]},{"id":"crm.tbl_cust_master.tin","service":"crm","table":"tbl_cust_master","column":"tin","type":"character varying(9)","verdict":"masked","class":"tax_id","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"tin\""],[0,"values","0 of 85 values are valid US ITINs"]],"shapes":[["999999999",100]],"examples":{"redact":"••••••••","last4":"•••••3525","null":null}},{"id":"crm.tbl_cust_master.usr_pwd_hash","service":"crm","table":"tbl_cust_master","column":"usr_pwd_hash","type":"character varying(100)","verdict":"masked","class":"credential","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"usr_pwd_hash\""],[75,"values","256 of 256 values are password hashes"]],"shapes":[["$9a$99$.A.A9a+9Aa+/99a+99Aa+9A.a+9a+9",0],["$9a$99$.a+9A9Aa+9Aa+9Aa+9a+",0],["$9a$99$.a+9a+9a+9A+/a+99999a+9a",0]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.tbl_cust_master.zip","service":"crm","table":"tbl_cust_master","column":"zip","type":"character varying(10)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"zip\""],[20,"values","209 of 256 values look like postal codes"]],"shapes":[["99999",81],["99999-9999",18]],"examples":{"redact":"••••••••","token":"ZIP_6ec9adf5","null":null}},{"id":"crm.tenant_settings.address_validation_provider","service":"crm","table":"tenant_settings","column":"address_validation_provider","type":"character varying(20)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"address_validation_provider\" in a person table"],[0,"values","0 of 220 values look like street addresses (few distinct values)"]],"shapes":[["a+",100]],"examples":{"redact":"••••••••","token":"ADDR_46ebba44","null":null}},{"id":"crm.tenant_settings.card_template","service":"crm","table":"tenant_settings","column":"card_template","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"card_template\""],[0,"values","0 of 220 values pass the card checksum (few distinct values)"]]},{"id":"crm.tenant_settings.created_at","service":"crm","table":"tenant_settings","column":"created_at","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"created_at\" matches no class"]]},{"id":"crm.tenant_settings.default_locale","service":"crm","table":"tenant_settings","column":"default_locale","type":"character varying(5)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"default_locale\" matches no class"]]},{"id":"crm.tenant_settings.dob_format","service":"crm","table":"tenant_settings","column":"dob_format","type":"character varying(12)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[15,"name","Column name \"dob_format\""],[0,"values","Values don't look like birth dates"]]},{"id":"crm.tenant_settings.email_domain_allowlist","service":"crm","table":"tenant_settings","column":"email_domain_allowlist","type":"character varying(120)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"email_domain_allowlist\" matches no class"],[0,"context","\"email domain\" rules out email"]]},{"id":"crm.tenant_settings.ip_whitelist_cidr","service":"crm","table":"tenant_settings","column":"ip_whitelist_cidr","type":"character varying(120)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ip_whitelist_cidr\" matches no class"],[0,"context","\"ip whitelist\" rules out ip address"]]},{"id":"crm.tenant_settings.mfa_required","service":"crm","table":"tenant_settings","column":"mfa_required","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"mfa_required\" matches no class"]]},{"id":"crm.tenant_settings.passport_strategy","service":"crm","table":"tenant_settings","column":"passport_strategy","type":"character varying(20)","verdict":"masked","class":"passport_number","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"passport_strategy\""],[0,"values","36 of 220 values look like passport numbers (few distinct values)"]],"shapes":[["a+",64],["a+-a+",19],["a+9",16]],"examples":{"redact":"••••••••","last4":"••••••••","null":null}},{"id":"crm.tenant_settings.password_expiry_days","service":"crm","table":"tenant_settings","column":"password_expiry_days","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"password_expiry_days\""],[0,"context","A smallint column can't hold credential"]]},{"id":"crm.tenant_settings.password_min_length","service":"crm","table":"tenant_settings","column":"password_min_length","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"password_min_length\" matches no class"],[0,"context","A smallint column can't hold credential"]]},{"id":"crm.tenant_settings.password_policy","service":"crm","table":"tenant_settings","column":"password_policy","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"password_policy\""],[0,"values","43 of 220 values look like secrets (few distinct values)"]]},{"id":"crm.tenant_settings.scorecard_weight","service":"crm","table":"tenant_settings","column":"scorecard_weight","type":"numeric(4,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"scorecard_weight\" matches no class"]]},{"id":"crm.tenant_settings.ssn_capture_enabled","service":"crm","table":"tenant_settings","column":"ssn_capture_enabled","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ssn_capture_enabled\" matches no class"],[0,"context","A boolean column can't hold national id"]]},{"id":"crm.tenant_settings.statement_day","service":"crm","table":"tenant_settings","column":"statement_day","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"statement_day\" matches no class"]]},{"id":"crm.tenant_settings.tenant_id","service":"crm","table":"tenant_settings","column":"tenant_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"tenant_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.tenant_settings.tenant_name","service":"crm","table":"tenant_settings","column":"tenant_name","type":"character varying(80)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"tenant_name\" in a person table"],[0,"values","7% of values are known names"]],"shapes":[["Aa+ Aa+ A+ 999",33],["Aa+ Aa+ Aa+ 999",19],["Aa+ Aa+ Aa+ 99",17]],"examples":{"token":"NAME_25b9715e","initial":"T••••• S••••••","redact":"••••••••","null":null}},{"id":"crm.user_accounts.avatarUrl","service":"crm","table":"user_accounts","column":"avatarUrl","type":"character varying(200)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"avatarUrl\" matches no class"]]},{"id":"crm.user_accounts.createdAt","service":"crm","table":"user_accounts","column":"createdAt","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"createdAt\" matches no class"]]},{"id":"crm.user_accounts.displayName","service":"crm","table":"user_accounts","column":"displayName","type":"character varying(80)","verdict":"masked","class":"person_name","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"displayName\" in a person table"],[45,"values","91% of values are known names"]],"shapes":[["Aa+ Aa+",54],["Aa+ A.",44],["Aa+ A'Aa+",0]],"examples":{"token":"NAME_8e61576f","initial":"A•••••• A••••","redact":"••••••••","null":null}},{"id":"crm.user_accounts.emailAddress","service":"crm","table":"user_accounts","column":"emailAddress","type":"character varying(120)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"emailAddress\""],[0,"context","\"email address\" rules out street address"],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["a+@a+.a+",25],["a+_a+99@a+.a+",16],["a+.a+99@a+.a+",15]],"examples":{"token":"EMAIL_05af4d9f","initial":"a••••••••@•••.com","redact":"••••••••","null":null}},{"id":"crm.user_accounts.isActive","service":"crm","table":"user_accounts","column":"isActive","type":"boolean","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"isActive\" matches no class"]]},{"id":"crm.user_accounts.lastLoginIp","service":"crm","table":"user_accounts","column":"lastLoginIp","type":"character varying(45)","verdict":"masked","class":"ip_address","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"lastLoginIp\""],[0,"context","\"last login\" rules out username"],[35,"values","200 of 256 values are IPv4 addresses"]],"shapes":[["999.999.999.99",7],["99.999.99.999",6],["99.999.999.999",5]],"examples":{"token":"IP_897ddb59","redact":"••••••••","null":null}},{"id":"crm.user_accounts.lastSeenAt","service":"crm","table":"user_accounts","column":"lastSeenAt","type":"timestamp with time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"lastSeenAt\" matches no class"]]},{"id":"crm.user_accounts.passwordHash","service":"crm","table":"user_accounts","column":"passwordHash","type":"character varying(120)","verdict":"masked","class":"credential","tier":"restricted","confidence":100,"total":100,"lines":[[70,"name","Column name \"passwordHash\""],[75,"values","256 of 256 values are password hashes"]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.user_accounts.preferredLocale","service":"crm","table":"user_accounts","column":"preferredLocale","type":"character varying(5)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"preferredLocale\" matches no class"]]},{"id":"crm.user_accounts.signupSource","service":"crm","table":"user_accounts","column":"signupSource","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"signupSource\" matches no class"]]},{"id":"crm.user_accounts.timeZone","service":"crm","table":"user_accounts","column":"timeZone","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"timeZone\" matches no class"]]},{"id":"crm.user_accounts.twoFactorSecret","service":"crm","table":"user_accounts","column":"twoFactorSecret","type":"character varying(64)","verdict":"masked","class":"credential","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"twoFactorSecret\""],[0,"values","0 of 106 values are password hashes"]],"shapes":[["A+9A+9A+9A+9A+",5],["A+99A+9A+9A+",1],["A+9A+99A+9A+",1]],"examples":{"redact":"••••••••","null":null}},{"id":"crm.user_accounts.userId","service":"crm","table":"user_accounts","column":"userId","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"userId\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"crm.user_accounts.userName","service":"crm","table":"user_accounts","column":"userName","type":"character varying(40)","verdict":"masked","class":"username","tier":"identity","confidence":90,"total":90,"lines":[[70,"name","Column name \"userName\""],[0,"context","\"user name\" rules out person name"],[20,"values","256 of 256 values look like usernames"]],"shapes":[["a+99",49],["a+.a+",22],["a+_999",19]],"examples":{"token":"USER_63b0ec15","initial":"a••••••••••••","redact":"••••••••","null":null}},{"id":"sports.addresses.building","service":"sports","table":"addresses","column":"building","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"building\" matches no class"]]},{"id":"sports.addresses.country","service":"sports","table":"addresses","column":"country","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"country\" matches no class"]]},{"id":"sports.addresses.county","service":"sports","table":"addresses","column":"county","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"county\" matches no class"]]},{"id":"sports.addresses.district","service":"sports","table":"addresses","column":"district","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"district\" matches no class"]]},{"id":"sports.addresses.floor","service":"sports","table":"addresses","column":"floor","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"floor\" matches no class"]]},{"id":"sports.addresses.id","service":"sports","table":"addresses","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.addresses.language","service":"sports","table":"addresses","column":"language","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"language\" matches no class"]]},{"id":"sports.addresses.locality","service":"sports","table":"addresses","column":"locality","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"locality\" matches no class"]]},{"id":"sports.addresses.location_id","service":"sports","table":"addresses","column":"location_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"location_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.addresses.neighborhood","service":"sports","table":"addresses","column":"neighborhood","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"neighborhood\" matches no class"]]},{"id":"sports.addresses.postal_code","service":"sports","table":"addresses","column":"postal_code","type":"character varying(100)","verdict":"masked","class":"postal_code","tier":"quasi","confidence":70,"total":70,"lines":[[70,"name","Column name \"postal_code\""]],"examples":{"redact":"••••••••","token":"ZIP_b34e2b5c","null":null}},{"id":"sports.addresses.region","service":"sports","table":"addresses","column":"region","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"region\" matches no class"]]},{"id":"sports.addresses.street","service":"sports","table":"addresses","column":"street","type":"character varying(100)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"street\""]],"examples":{"redact":"••••••••","token":"ADDR_3ad1c580","null":null}},{"id":"sports.addresses.street_number","service":"sports","table":"addresses","column":"street_number","type":"character varying(100)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"street_number\""]],"examples":{"redact":"••••••••","token":"ADDR_3ad1c580","null":null}},{"id":"sports.addresses.street_prefix","service":"sports","table":"addresses","column":"street_prefix","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"street_prefix\""]]},{"id":"sports.addresses.street_suffix","service":"sports","table":"addresses","column":"street_suffix","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"street_suffix\""]]},{"id":"sports.addresses.suite","service":"sports","table":"addresses","column":"suite","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"suite\" matches no class"]]},{"id":"sports.affiliation_phases.affiliation_id","service":"sports","table":"affiliation_phases","column":"affiliation_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"affiliation_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliation_phases.ancestor_affiliation_id","service":"sports","table":"affiliation_phases","column":"ancestor_affiliation_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"ancestor_affiliation_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliation_phases.end_date_time","service":"sports","table":"affiliation_phases","column":"end_date_time","type":"timestamp without time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"end_date_time\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.affiliation_phases.end_season_id","service":"sports","table":"affiliation_phases","column":"end_season_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"end_season_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliation_phases.id","service":"sports","table":"affiliation_phases","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliation_phases.start_date_time","service":"sports","table":"affiliation_phases","column":"start_date_time","type":"timestamp without time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"start_date_time\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.affiliation_phases.start_season_id","service":"sports","table":"affiliation_phases","column":"start_season_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"start_season_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations.affiliation_key","service":"sports","table":"affiliations","column":"affiliation_key","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"affiliation_key\""],[0,"values","0 of 29 values are password hashes"]]},{"id":"sports.affiliations.affiliation_type","service":"sports","table":"affiliations","column":"affiliation_type","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"affiliation_type\" matches no class"]]},{"id":"sports.affiliations.id","service":"sports","table":"affiliations","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations.publisher_id","service":"sports","table":"affiliations","column":"publisher_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"publisher_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations_documents.affiliation_id","service":"sports","table":"affiliations_documents","column":"affiliation_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"affiliation_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations_documents.document_id","service":"sports","table":"affiliations_documents","column":"document_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"document_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations_events.affiliation_id","service":"sports","table":"affiliations_events","column":"affiliation_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"affiliation_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations_events.event_id","service":"sports","table":"affiliations_events","column":"event_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"event_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations_media.affiliation_id","service":"sports","table":"affiliations_media","column":"affiliation_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"affiliation_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.affiliations_media.media_id","service":"sports","table":"affiliations_media","column":"media_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"media_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_action_participants.american_football_action_play_id","service":"sports","table":"american_football_action_participants","column":"american_football_action_play_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"american_football_action_play_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_action_participants.field_line","service":"sports","table":"american_football_action_participants","column":"field_line","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"field_line\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_action_participants.id","service":"sports","table":"american_football_action_participants","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_action_participants.participant_role","service":"sports","table":"american_football_action_participants","column":"participant_role","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"participant_role\" matches no class"]]},{"id":"sports.american_football_action_participants.person_id","service":"sports","table":"american_football_action_participants","column":"person_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"person_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_action_participants.score_credit","service":"sports","table":"american_football_action_participants","column":"score_credit","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"score_credit\" matches no class"]]},{"id":"sports.american_football_action_participants.score_type","service":"sports","table":"american_football_action_participants","column":"score_type","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"score_type\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_action_participants.yardage","service":"sports","table":"american_football_action_participants","column":"yardage","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"yardage\" matches no class"]]},{"id":"sports.american_football_action_participants.yards_gained","service":"sports","table":"american_football_action_participants","column":"yards_gained","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"yards_gained\" matches no class"]]},{"id":"sports.american_football_action_plays.american_football_event_state_id","service":"sports","table":"american_football_action_plays","column":"american_football_event_state_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"american_football_event_state_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_action_plays.comment","service":"sports","table":"american_football_action_plays","column":"comment","type":"character varying(255)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"comment\" matches no class"]]},{"id":"sports.american_football_action_plays.drive_result","service":"sports","table":"american_football_action_plays","column":"drive_result","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"drive_result\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_action_plays.id","service":"sports","table":"american_football_action_plays","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_action_plays.play_type","service":"sports","table":"american_football_action_plays","column":"play_type","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"play_type\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_action_plays.points","service":"sports","table":"american_football_action_plays","column":"points","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"points\" matches no class"]]},{"id":"sports.american_football_action_plays.score_attempt_type","service":"sports","table":"american_football_action_plays","column":"score_attempt_type","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"score_attempt_type\" matches no class"]]},{"id":"sports.american_football_defensive_stats.id","service":"sports","table":"american_football_defensive_stats","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_defensive_stats.interceptions_average","service":"sports","table":"american_football_defensive_stats","column":"interceptions_average","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"interceptions_average\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_defensive_stats.interceptions_longest","service":"sports","table":"american_football_defensive_stats","column":"interceptions_longest","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"interceptions_longest\" matches no class"]]},{"id":"sports.american_football_defensive_stats.interceptions_total","service":"sports","table":"american_football_defensive_stats","column":"interceptions_total","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"interceptions_total\" matches no class"]]},{"id":"sports.american_football_defensive_stats.interceptions_touchdown","service":"sports","table":"american_football_defensive_stats","column":"interceptions_touchdown","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"interceptions_touchdown\" matches no class"]]},{"id":"sports.american_football_defensive_stats.interceptions_yards","service":"sports","table":"american_football_defensive_stats","column":"interceptions_yards","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"interceptions_yards\" matches no class"]]},{"id":"sports.american_football_defensive_stats.passes_defensed","service":"sports","table":"american_football_defensive_stats","column":"passes_defensed","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_defensed\""],[0,"values","0 of 246 values are password hashes (few distinct values)"]]},{"id":"sports.american_football_defensive_stats.quarterback_hurries","service":"sports","table":"american_football_defensive_stats","column":"quarterback_hurries","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"quarterback_hurries\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_defensive_stats.sacks_total","service":"sports","table":"american_football_defensive_stats","column":"sacks_total","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sacks_total\" matches no class"]]},{"id":"sports.american_football_defensive_stats.sacks_yards","service":"sports","table":"american_football_defensive_stats","column":"sacks_yards","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sacks_yards\" matches no class"]]},{"id":"sports.american_football_defensive_stats.tackles_assists","service":"sports","table":"american_football_defensive_stats","column":"tackles_assists","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"tackles_assists\" matches no class"]]},{"id":"sports.american_football_defensive_stats.tackles_solo","service":"sports","table":"american_football_defensive_stats","column":"tackles_solo","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"tackles_solo\" matches no class"]]},{"id":"sports.american_football_defensive_stats.tackles_total","service":"sports","table":"american_football_defensive_stats","column":"tackles_total","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"tackles_total\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.conversions_fourth_down","service":"sports","table":"american_football_down_progress_stats","column":"conversions_fourth_down","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"conversions_fourth_down\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.conversions_fourth_down_attempts","service":"sports","table":"american_football_down_progress_stats","column":"conversions_fourth_down_attempts","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"conversions_fourth_down_attempts\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.conversions_fourth_down_percentage","service":"sports","table":"american_football_down_progress_stats","column":"conversions_fourth_down_percentage","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"conversions_fourth_down_percentage\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.conversions_third_down","service":"sports","table":"american_football_down_progress_stats","column":"conversions_third_down","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"conversions_third_down\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.conversions_third_down_attempts","service":"sports","table":"american_football_down_progress_stats","column":"conversions_third_down_attempts","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"conversions_third_down_attempts\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.conversions_third_down_percentage","service":"sports","table":"american_football_down_progress_stats","column":"conversions_third_down_percentage","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"conversions_third_down_percentage\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.first_downs_pass","service":"sports","table":"american_football_down_progress_stats","column":"first_downs_pass","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"first_downs_pass\""],[0,"values","0 of 26 values are password hashes"]]},{"id":"sports.american_football_down_progress_stats.first_downs_penalty","service":"sports","table":"american_football_down_progress_stats","column":"first_downs_penalty","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"first_downs_penalty\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.first_downs_run","service":"sports","table":"american_football_down_progress_stats","column":"first_downs_run","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"first_downs_run\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.first_downs_total","service":"sports","table":"american_football_down_progress_stats","column":"first_downs_total","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"first_downs_total\" matches no class"]]},{"id":"sports.american_football_down_progress_stats.id","service":"sports","table":"american_football_down_progress_stats","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_event_states.clock_state","service":"sports","table":"american_football_event_states","column":"clock_state","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"clock_state\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_event_states.context","service":"sports","table":"american_football_event_states","column":"context","type":"character varying(40)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"context\" matches no class"]]},{"id":"sports.american_football_event_states.current_state","service":"sports","table":"american_football_event_states","column":"current_state","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"current_state\" matches no class"]]},{"id":"sports.american_football_event_states.distance_for_1st_down","service":"sports","table":"american_football_event_states","column":"distance_for_1st_down","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"distance_for_1st_down\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_event_states.down","service":"sports","table":"american_football_event_states","column":"down","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"down\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_event_states.event_id","service":"sports","table":"american_football_event_states","column":"event_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"event_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_event_states.field_line","service":"sports","table":"american_football_event_states","column":"field_line","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"field_line\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_event_states.field_side","service":"sports","table":"american_football_event_states","column":"field_side","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"field_side\" matches no class"]]},{"id":"sports.american_football_event_states.id","service":"sports","table":"american_football_event_states","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_event_states.period_time_elapsed","service":"sports","table":"american_football_event_states","column":"period_time_elapsed","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"period_time_elapsed\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_event_states.period_time_remaining","service":"sports","table":"american_football_event_states","column":"period_time_remaining","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"period_time_remaining\" matches no class"]]},{"id":"sports.american_football_event_states.period_value","service":"sports","table":"american_football_event_states","column":"period_value","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"period_value\" matches no class"]]},{"id":"sports.american_football_event_states.sequence_number","service":"sports","table":"american_football_event_states","column":"sequence_number","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sequence_number\" matches no class"]]},{"id":"sports.american_football_event_states.team_in_possession_id","service":"sports","table":"american_football_event_states","column":"team_in_possession_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"team_in_possession_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_fumbles_stats.fumbles_committed","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_committed","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_committed\" matches no class"]]},{"id":"sports.american_football_fumbles_stats.fumbles_forced","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_forced","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_forced\" matches no class"]]},{"id":"sports.american_football_fumbles_stats.fumbles_lost","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_lost","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_lost\" matches no class"]]},{"id":"sports.american_football_fumbles_stats.fumbles_opposing_committed","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_opposing_committed","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_opposing_committed\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_fumbles_stats.fumbles_opposing_lost","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_opposing_lost","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_opposing_lost\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_fumbles_stats.fumbles_opposing_recovered","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_opposing_recovered","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_opposing_recovered\" matches no class"]]},{"id":"sports.american_football_fumbles_stats.fumbles_opposing_yards_gained","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_opposing_yards_gained","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_opposing_yards_gained\" matches no class"]]},{"id":"sports.american_football_fumbles_stats.fumbles_own_committed","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_own_committed","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_own_committed\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_fumbles_stats.fumbles_own_lost","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_own_lost","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_own_lost\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_fumbles_stats.fumbles_own_recovered","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_own_recovered","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_own_recovered\" matches no class"]]},{"id":"sports.american_football_fumbles_stats.fumbles_own_yards_gained","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_own_yards_gained","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_own_yards_gained\" matches no class"]]},{"id":"sports.american_football_fumbles_stats.fumbles_recovered","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_recovered","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_recovered\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_fumbles_stats.fumbles_yards_gained","service":"sports","table":"american_football_fumbles_stats","column":"fumbles_yards_gained","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"fumbles_yards_gained\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_fumbles_stats.id","service":"sports","table":"american_football_fumbles_stats","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_offensive_stats.id","service":"sports","table":"american_football_offensive_stats","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_offensive_stats.offensive_plays_average_yards_per","service":"sports","table":"american_football_offensive_stats","column":"offensive_plays_average_yards_per","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"offensive_plays_average_yards_per\" matches no class"]]},{"id":"sports.american_football_offensive_stats.offensive_plays_number","service":"sports","table":"american_football_offensive_stats","column":"offensive_plays_number","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"offensive_plays_number\" matches no class"]]},{"id":"sports.american_football_offensive_stats.offensive_plays_yards","service":"sports","table":"american_football_offensive_stats","column":"offensive_plays_yards","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"offensive_plays_yards\" matches no class"]]},{"id":"sports.american_football_offensive_stats.possession_duration","service":"sports","table":"american_football_offensive_stats","column":"possession_duration","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"possession_duration\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_offensive_stats.turnovers_giveaway","service":"sports","table":"american_football_offensive_stats","column":"turnovers_giveaway","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"turnovers_giveaway\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_passing_stats.id","service":"sports","table":"american_football_passing_stats","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_passing_stats.passer_rating","service":"sports","table":"american_football_passing_stats","column":"passer_rating","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"passer_rating\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_passing_stats.passes_attempts","service":"sports","table":"american_football_passing_stats","column":"passes_attempts","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"passes_attempts\""],[0,"values","0 of 56 values are password hashes"]]},{"id":"sports.american_football_passing_stats.passes_average_yards_per","service":"sports","table":"american_football_passing_stats","column":"passes_average_yards_per","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_average_yards_per\""],[0,"values","0 of 26 values are password hashes"]]},{"id":"sports.american_football_passing_stats.passes_completions","service":"sports","table":"american_football_passing_stats","column":"passes_completions","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_completions\""],[0,"values","0 of 56 values are password hashes"]]},{"id":"sports.american_football_passing_stats.passes_interceptions","service":"sports","table":"american_football_passing_stats","column":"passes_interceptions","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_interceptions\""],[0,"values","0 of 56 values are password hashes (few distinct values)"]]},{"id":"sports.american_football_passing_stats.passes_interceptions_percentage","service":"sports","table":"american_football_passing_stats","column":"passes_interceptions_percentage","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_interceptions_percentage\""],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_passing_stats.passes_longest","service":"sports","table":"american_football_passing_stats","column":"passes_longest","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_longest\""],[0,"values","0 of 30 values are password hashes"]]},{"id":"sports.american_football_passing_stats.passes_percentage","service":"sports","table":"american_football_passing_stats","column":"passes_percentage","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_percentage\""],[0,"values","0 of 30 values are password hashes"]]},{"id":"sports.american_football_passing_stats.passes_touchdowns","service":"sports","table":"american_football_passing_stats","column":"passes_touchdowns","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_touchdowns\""],[0,"values","0 of 30 values are password hashes (few distinct values)"]]},{"id":"sports.american_football_passing_stats.passes_touchdowns_percentage","service":"sports","table":"american_football_passing_stats","column":"passes_touchdowns_percentage","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_touchdowns_percentage\""],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_passing_stats.passes_yards_gross","service":"sports","table":"american_football_passing_stats","column":"passes_yards_gross","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_yards_gross\""],[0,"values","0 of 56 values are password hashes"]]},{"id":"sports.american_football_passing_stats.passes_yards_lost","service":"sports","table":"american_football_passing_stats","column":"passes_yards_lost","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_yards_lost\""],[0,"values","0 of 26 values are password hashes"]]},{"id":"sports.american_football_passing_stats.passes_yards_net","service":"sports","table":"american_football_passing_stats","column":"passes_yards_net","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"passes_yards_net\""],[0,"values","0 of 26 values are password hashes"]]},{"id":"sports.american_football_passing_stats.receptions_average_yards_per","service":"sports","table":"american_football_passing_stats","column":"receptions_average_yards_per","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"receptions_average_yards_per\" matches no class"]]},{"id":"sports.american_football_passing_stats.receptions_first_down","service":"sports","table":"american_football_passing_stats","column":"receptions_first_down","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"receptions_first_down\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_passing_stats.receptions_longest","service":"sports","table":"american_football_passing_stats","column":"receptions_longest","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"receptions_longest\" matches no class"]]},{"id":"sports.american_football_passing_stats.receptions_total","service":"sports","table":"american_football_passing_stats","column":"receptions_total","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"receptions_total\" matches no class"]]},{"id":"sports.american_football_passing_stats.receptions_touchdowns","service":"sports","table":"american_football_passing_stats","column":"receptions_touchdowns","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"receptions_touchdowns\" matches no class"]]},{"id":"sports.american_football_passing_stats.receptions_yards","service":"sports","table":"american_football_passing_stats","column":"receptions_yards","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"receptions_yards\" matches no class"]]},{"id":"sports.american_football_penalties_stats.id","service":"sports","table":"american_football_penalties_stats","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_penalties_stats.penalties_total","service":"sports","table":"american_football_penalties_stats","column":"penalties_total","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"penalties_total\" matches no class"]]},{"id":"sports.american_football_penalties_stats.penalty_first_downs","service":"sports","table":"american_football_penalties_stats","column":"penalty_first_downs","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"penalty_first_downs\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_penalties_stats.penalty_yards","service":"sports","table":"american_football_penalties_stats","column":"penalty_yards","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"penalty_yards\" matches no class"]]},{"id":"sports.american_football_rushing_stats.id","service":"sports","table":"american_football_rushing_stats","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.american_football_rushing_stats.rushes_attempts","service":"sports","table":"american_football_rushing_stats","column":"rushes_attempts","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rushes_attempts\" matches no class"]]},{"id":"sports.american_football_rushing_stats.rushes_first_down","service":"sports","table":"american_football_rushing_stats","column":"rushes_first_down","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rushes_first_down\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.american_football_rushing_stats.rushes_longest","service":"sports","table":"american_football_rushing_stats","column":"rushes_longest","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rushes_longest\" matches no class"]]},{"id":"sports.american_football_rushing_stats.rushes_touchdowns","service":"sports","table":"american_football_rushing_stats","column":"rushes_touchdowns","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rushes_touchdowns\" matches no class"]]},{"id":"sports.american_football_rushing_stats.rushes_yards","service":"sports","table":"american_football_rushing_stats","column":"rushes_yards","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rushes_yards\" matches no class"]]},{"id":"sports.american_football_rushing_stats.rushing_average_yards_per","service":"sports","table":"american_football_rushing_stats","column":"rushing_average_yards_per","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"rushing_average_yards_per\" matches no class"]]},{"id":"sports.display_names.abbreviation","service":"sports","table":"display_names","column":"abbreviation","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"abbreviation\" matches no class"]]},{"id":"sports.display_names.alias","service":"sports","table":"display_names","column":"alias","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":35,"total":35,"lines":[[15,"name","Column name \"alias\""],[20,"values","89 of 102 values look like usernames"]]},{"id":"sports.display_names.entity_id","service":"sports","table":"display_names","column":"entity_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"entity_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.display_names.entity_type","service":"sports","table":"display_names","column":"entity_type","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"entity_type\" matches no class"]]},{"id":"sports.display_names.first_name","service":"sports","table":"display_names","column":"first_name","type":"character varying(100)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"first_name\""],[0,"values","28% of values are known names"]],"shapes":[["Aa+",85],["Aa+ Aa+",11],["A.A.",1]],"examples":{"token":"NAME_95674350","initial":"J•••","redact":"••••••••","null":null}},{"id":"sports.display_names.full_name","service":"sports","table":"display_names","column":"full_name","type":"character varying(100)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"full_name\""],[0,"values","32% of values are known names"]],"shapes":[["Aa+ Aa+",75],["Aa+ Aa+ Aa+",12],["Aa+",4]],"examples":{"token":"NAME_2155b5c0","initial":"M•••• L••••• B•••••••","redact":"••••••••","null":null}},{"id":"sports.display_names.id","service":"sports","table":"display_names","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.display_names.language","service":"sports","table":"display_names","column":"language","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"language\" matches no class"]]},{"id":"sports.display_names.last_name","service":"sports","table":"display_names","column":"last_name","type":"character varying(100)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"last_name\""],[0,"values","21% of values are known names"]],"shapes":[["Aa+",96],["Aa+ Aa+",2],["99a+",0]],"examples":{"token":"NAME_014e9955","initial":"S••••••","redact":"••••••••","null":null}},{"id":"sports.display_names.middle_name","service":"sports","table":"display_names","column":"middle_name","type":"character varying(100)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"middle_name\""],[0,"context","Only 0 non-null values sampled"]],"examples":{"token":"NAME_c0b3ed2e","initial":"J••• D••","redact":"••••••••","null":null}},{"id":"sports.display_names.prefix","service":"sports","table":"display_names","column":"prefix","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"prefix\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.display_names.short_name","service":"sports","table":"display_names","column":"short_name","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"short_name\" matches no class"],[0,"context","\"short name\" rules out person name"]]},{"id":"sports.display_names.suffix","service":"sports","table":"display_names","column":"suffix","type":"character varying(20)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"suffix\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.injury_phases.disabled_list","service":"sports","table":"injury_phases","column":"disabled_list","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"disabled_list\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.injury_phases.end_date_time","service":"sports","table":"injury_phases","column":"end_date_time","type":"timestamp without time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"end_date_time\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.injury_phases.id","service":"sports","table":"injury_phases","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.injury_phases.injury_comment","service":"sports","table":"injury_phases","column":"injury_comment","type":"character varying(100)","verdict":"review","class":"special_category","tier":"restricted","confidence":55,"total":55,"lines":[[55,"name","Column name \"injury_comment\""],[0,"context","Only 0 non-null values sampled"]],"examples":{"redact":"••••••••","null":null}},{"id":"sports.injury_phases.injury_side","service":"sports","table":"injury_phases","column":"injury_side","type":"character varying(100)","verdict":"review","class":"special_category","tier":"restricted","confidence":55,"total":55,"lines":[[55,"name","Column name \"injury_side\""],[0,"values","0 of 130 values are ICD-10 diagnosis codes (few distinct values)"]],"shapes":[["a+",100]],"examples":{"redact":"••••••••","null":null}},{"id":"sports.injury_phases.injury_status","service":"sports","table":"injury_phases","column":"injury_status","type":"character varying(100)","verdict":"review","class":"special_category","tier":"restricted","confidence":55,"total":55,"lines":[[55,"name","Column name \"injury_status\""],[0,"context","Only 0 non-null values sampled"]],"examples":{"redact":"••••••••","null":null}},{"id":"sports.injury_phases.injury_type","service":"sports","table":"injury_phases","column":"injury_type","type":"character varying(100)","verdict":"review","class":"special_category","tier":"restricted","confidence":55,"total":55,"lines":[[55,"name","Column name \"injury_type\""],[0,"values","0 of 256 values are ICD-10 diagnosis codes"]],"shapes":[["a+",87],["a+-a+",12]],"examples":{"redact":"••••••••","null":null}},{"id":"sports.injury_phases.person_id","service":"sports","table":"injury_phases","column":"person_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"person_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.injury_phases.phase_type","service":"sports","table":"injury_phases","column":"phase_type","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"phase_type\" matches no class"]]},{"id":"sports.injury_phases.season_id","service":"sports","table":"injury_phases","column":"season_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"season_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.injury_phases.start_date_time","service":"sports","table":"injury_phases","column":"start_date_time","type":"timestamp without time zone","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"start_date_time\" matches no class"]]},{"id":"sports.persons.birth_date","service":"sports","table":"persons","column":"birth_date","type":"character varying(30)","verdict":"masked","class":"date_of_birth","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"birth_date\""],[0,"context","Only 0 non-null values sampled"]],"examples":{"year":"1984-••-••","redact":"••••••••","token":"DOB_48287967","null":null}},{"id":"sports.persons.birth_location_id","service":"sports","table":"persons","column":"birth_location_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"birth_location_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.persons.death_date","service":"sports","table":"persons","column":"death_date","type":"character varying(30)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"death_date\" matches no class"],[0,"context","Only 0 non-null values sampled"]]},{"id":"sports.persons.death_location_id","service":"sports","table":"persons","column":"death_location_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"death_location_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.persons.gender","service":"sports","table":"persons","column":"gender","type":"character varying(20)","verdict":"masked","class":"gender","tier":"quasi","confidence":70,"total":70,"lines":[[70,"name","Column name \"gender\""],[0,"context","Only 0 non-null values sampled"]],"examples":{"redact":"••••••••","token":"GENDER_82f007d8","null":null}},{"id":"sports.persons.hometown_location_id","service":"sports","table":"persons","column":"hometown_location_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"hometown_location_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.persons.id","service":"sports","table":"persons","column":"id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.persons.person_key","service":"sports","table":"persons","column":"person_key","type":"character varying(100)","verdict":"clear","class":null,"tier":null,"confidence":15,"total":15,"lines":[[15,"name","Column name \"person_key\""],[0,"values","0 of 256 values are password hashes"]]},{"id":"sports.persons.publisher_id","service":"sports","table":"persons","column":"publisher_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"publisher_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"sports.persons.residence_location_id","service":"sports","table":"persons","column":"residence_location_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"residence_location_id\" matches no class"],[0,"context","\"residence location id\" rules out street address"]]},{"id":"store.categories.category","service":"store","table":"categories","column":"category","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"category\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.categories.categoryname","service":"store","table":"categories","column":"categoryname","type":"character varying(50)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"categoryname\" matches no class"],[0,"context","\"category name\" rules out person name"]]},{"id":"store.cust_hist.customerid","service":"store","table":"cust_hist","column":"customerid","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customerid\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.cust_hist.orderid","service":"store","table":"cust_hist","column":"orderid","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"orderid\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.cust_hist.prod_id","service":"store","table":"cust_hist","column":"prod_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"prod_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.customers.address1","service":"store","table":"customers","column":"address1","type":"character varying(50)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"address1\""],[0,"values","0 of 256 values look like street addresses"]],"shapes":[["9999999999 Aa+ Aa+",100]],"examples":{"redact":"••••••••","token":"ADDR_ddaf9d9e","null":null}},{"id":"store.customers.address2","service":"store","table":"customers","column":"address2","type":"character varying(50)","verdict":"masked","class":"street_address","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"address2\""],[0,"context","Only 0 non-null values sampled"]],"examples":{"redact":"••••••••","token":"ADDR_3ad1c580","null":null}},{"id":"store.customers.age","service":"store","table":"customers","column":"age","type":"smallint","verdict":"masked","class":"age","tier":"quasi","confidence":90,"total":90,"lines":[[70,"name","Column name \"age\" in a person table"],[20,"values","256 of 256 values are plausible ages"]],"shapes":[["99",100]],"examples":{"redact":"••••••••","null":null}},{"id":"store.customers.city","service":"store","table":"customers","column":"city","type":"character varying(50)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"city\" matches no class"]]},{"id":"store.customers.country","service":"store","table":"customers","column":"country","type":"character varying(50)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"country\" matches no class"]]},{"id":"store.customers.creditcard","service":"store","table":"customers","column":"creditcard","type":"character varying(50)","verdict":"masked","class":"card_number","tier":"restricted","confidence":90,"total":90,"lines":[[70,"name","Column name \"creditcard\""],[20,"values","256 of 256 values look like card numbers"]],"shapes":[["9999999999999999",100]],"examples":{"last4":"••••••••••••5911","redact":"••••••••","null":null}},{"id":"store.customers.creditcardexpiration","service":"store","table":"customers","column":"creditcardexpiration","type":"character varying(50)","verdict":"masked","class":"card_expiry","tier":"quasi","confidence":100,"total":100,"lines":[[70,"name","Column name \"creditcardexpiration\""],[45,"values","256 of 256 values look like card expiry dates"]],"shapes":[["9999/99",100]],"examples":{"redact":"••••••••","year":"2012/••","null":null}},{"id":"store.customers.creditcardtype","service":"store","table":"customers","column":"creditcardtype","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"creditcardtype\" matches no class"],[0,"context","\"card type\" rules out card number"]]},{"id":"store.customers.customerid","service":"store","table":"customers","column":"customerid","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customerid\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.customers.email","service":"store","table":"customers","column":"email","type":"character varying(50)","verdict":"masked","class":"email","tier":"identity","confidence":100,"total":100,"lines":[[70,"name","Column name \"email\""],[75,"values","256 of 256 values are valid email addresses"]],"shapes":[["A+@a+.a+",100]],"examples":{"token":"EMAIL_07fd7849","initial":"I•••••••••@••••.com","redact":"••••••••","null":null}},{"id":"store.customers.firstname","service":"store","table":"customers","column":"firstname","type":"character varying(50)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"firstname\""],[0,"values","0% of values are known names"]],"shapes":[["A+",100]],"examples":{"token":"NAME_e314bde1","initial":"V•••••","redact":"••••••••","null":null}},{"id":"store.customers.gender","service":"store","table":"customers","column":"gender","type":"character varying(1)","verdict":"masked","class":"gender","tier":"quasi","confidence":80,"total":80,"lines":[[70,"name","Column name \"gender\""],[10,"values","256 of 256 values are gender values (few distinct values)"]],"shapes":[["A",100]],"examples":{"redact":"••••••••","token":"GENDER_ea07bdeb","null":null}},{"id":"store.customers.income","service":"store","table":"customers","column":"income","type":"integer","verdict":"masked","class":"financial_personal","tier":"quasi","confidence":80,"total":80,"lines":[[70,"name","Column name \"income\" in a person table"],[10,"values","256 of 256 values are money amounts (few distinct values)"]],"shapes":[["99999",78],["999999",21]],"examples":{"redact":"••••••••","null":null}},{"id":"store.customers.lastname","service":"store","table":"customers","column":"lastname","type":"character varying(50)","verdict":"masked","class":"person_name","tier":"identity","confidence":70,"total":70,"lines":[[70,"name","Column name \"lastname\""],[0,"values","0% of values are known names"]],"shapes":[["A+",100]],"examples":{"token":"NAME_55f0b9c5","initial":"I•••••••••","redact":"••••••••","null":null}},{"id":"store.customers.password","service":"store","table":"customers","column":"password","type":"character varying(50)","verdict":"masked","class":"credential","tier":"restricted","confidence":70,"total":70,"lines":[[70,"name","Column name \"password\""],[0,"values","0 of 256 values are password hashes (few distinct values)"]],"shapes":[["a+",100]],"examples":{"redact":"••••••••","null":null}},{"id":"store.customers.phone","service":"store","table":"customers","column":"phone","type":"character varying(50)","verdict":"masked","class":"phone","tier":"identity","confidence":90,"total":90,"lines":[[70,"name","Column name \"phone\""],[20,"values","256 of 256 values are phone-length digit strings"]],"shapes":[["9999999999",100]],"examples":{"token":"PHONE_8f1c71cd","last4":"••••••9546","redact":"••••••••","null":null}},{"id":"store.customers.region","service":"store","table":"customers","column":"region","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"region\" matches no class"]]},{"id":"store.customers.state","service":"store","table":"customers","column":"state","type":"character varying(50)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"state\" matches no class"]]},{"id":"store.customers.username","service":"store","table":"customers","column":"username","type":"character varying(50)","verdict":"masked","class":"username","tier":"identity","confidence":90,"total":90,"lines":[[70,"name","Column name \"username\""],[20,"values","256 of 256 values look like usernames"]],"shapes":[["a+99999",50],["a+99",35],["a+999",11]],"examples":{"token":"USER_91eec095","initial":"u••••","redact":"••••••••","null":null}},{"id":"store.customers.zip","service":"store","table":"customers","column":"zip","type":"integer","verdict":"masked","class":"postal_code","tier":"quasi","confidence":70,"total":70,"lines":[[70,"name","Column name \"zip\""],[0,"values","128 of 256 values look like postal codes"]],"shapes":[["9",50],["99999",50]],"examples":{"redact":"••••••••","token":"ZIP_a8a6e751","null":null}},{"id":"store.inventory.prod_id","service":"store","table":"inventory","column":"prod_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"prod_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.inventory.quan_in_stock","service":"store","table":"inventory","column":"quan_in_stock","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"quan_in_stock\" matches no class"]]},{"id":"store.inventory.sales","service":"store","table":"inventory","column":"sales","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"sales\" matches no class"]]},{"id":"store.orderlines.orderdate","service":"store","table":"orderlines","column":"orderdate","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"orderdate\" matches no class"]]},{"id":"store.orderlines.orderid","service":"store","table":"orderlines","column":"orderid","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"orderid\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.orderlines.orderlineid","service":"store","table":"orderlines","column":"orderlineid","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"orderlineid\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.orderlines.prod_id","service":"store","table":"orderlines","column":"prod_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"prod_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.orderlines.quantity","service":"store","table":"orderlines","column":"quantity","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"quantity\" matches no class"]]},{"id":"store.orders.customerid","service":"store","table":"orders","column":"customerid","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"customerid\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.orders.netamount","service":"store","table":"orders","column":"netamount","type":"numeric(12,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"netamount\" matches no class"]]},{"id":"store.orders.orderdate","service":"store","table":"orders","column":"orderdate","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"orderdate\" matches no class"]]},{"id":"store.orders.orderid","service":"store","table":"orders","column":"orderid","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"orderid\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.orders.tax","service":"store","table":"orders","column":"tax","type":"numeric(12,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"tax\" matches no class"]]},{"id":"store.orders.totalamount","service":"store","table":"orders","column":"totalamount","type":"numeric(12,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"totalamount\" matches no class"]]},{"id":"store.products.actor","service":"store","table":"products","column":"actor","type":"character varying(50)","verdict":"clear","class":null,"tier":null,"confidence":35,"total":35,"lines":[[0,"name","Column name \"actor\" matches no class"],[35,"values","67% of values are known names"]]},{"id":"store.products.category","service":"store","table":"products","column":"category","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"category\" matches no class"]]},{"id":"store.products.common_prod_id","service":"store","table":"products","column":"common_prod_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"common_prod_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.products.price","service":"store","table":"products","column":"price","type":"numeric(12,2)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"price\" matches no class"]]},{"id":"store.products.prod_id","service":"store","table":"products","column":"prod_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"prod_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.products.special","service":"store","table":"products","column":"special","type":"smallint","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"special\" matches no class"]]},{"id":"store.products.title","service":"store","table":"products","column":"title","type":"character varying(50)","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"title\" matches no class"]]},{"id":"store.reorder.date_expected","service":"store","table":"reorder","column":"date_expected","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"date_expected\" matches no class"]]},{"id":"store.reorder.date_low","service":"store","table":"reorder","column":"date_low","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"date_low\" matches no class"]]},{"id":"store.reorder.date_reordered","service":"store","table":"reorder","column":"date_reordered","type":"date","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"date_reordered\" matches no class"]]},{"id":"store.reorder.prod_id","service":"store","table":"reorder","column":"prod_id","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"prod_id\" matches no class"],[0,"context","Key column, never classified"]]},{"id":"store.reorder.quan_low","service":"store","table":"reorder","column":"quan_low","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"quan_low\" matches no class"]]},{"id":"store.reorder.quan_reordered","service":"store","table":"reorder","column":"quan_reordered","type":"integer","verdict":"clear","class":null,"tier":null,"confidence":0,"total":0,"lines":[[0,"name","Column name \"quan_reordered\" matches no class"]]}]} \ No newline at end of file diff --git a/e2e/fixtures/data-masking/generate-fixture.ts b/e2e/fixtures/data-masking/generate-fixture.ts new file mode 100644 index 000000000..355a90e8b --- /dev/null +++ b/e2e/fixtures/data-masking/generate-fixture.ts @@ -0,0 +1,438 @@ +/** + * Builds e2e/fixtures/data-masking/fixture.json from real classifier output. + * + * Input: the classifier prototype's per-database JSON + * ({meta, columns: {"db.table.column": {verdict, class, tier, confidence, + * evidence: {why, score, shapes, type, ...}}}}). + * Output: a compact fixture the Playwright mock serves. Evidence is + * humanized ("+75 values 256/256 (100%) pass ssn_us.dashed (strong)" -> + * [75, "values", "256 of 256 values are valid US SSNs"]) and detector noise + * is dropped. Each sensitive column carries `examples`: the masked rendering + * of a REAL first-row value (fetched from a live DreamFactory) in every + * format its class allows. Raw values are never written. + * + * Run (Node 18+, from the repo root): + * npx esbuild e2e/fixtures/data-masking/generate-fixture.ts --bundle \ + * --platform=node --format=esm --outfile=/tmp/dm-gen.mjs + * DF_URL=http://127.0.0.1:8790 DF_ADMIN_EMAIL=... DF_ADMIN_PASSWORD=... \ + * node /tmp/dm-gen.mjs \ + * crm=/corpus.hybrid.json \ + * sports=/sportsdb.hybrid.json \ + * store=/dellstore.hybrid.json + */ +import { readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + CLASSES, + classDef, +} from '../../../src/app/adf-data-masking/data-masking.catalog'; +import { + applyMask, + maskSpans, +} from '../../../src/app/adf-data-masking/mask-format'; + +type Line = [number, string, string]; + +const DETECTORS: Record = { + 'email.syntax': 'are valid email addresses', + 'phone.formatted': 'are formatted phone numbers', + 'phone.digits': 'are phone-length digit strings', + 'street.pattern': 'look like street addresses', + 'postal.numeric': 'look like postal codes', + 'postal.us_zip4': 'are US ZIP+4 codes', + 'postal.nl': 'are Dutch postcodes', + 'postal.uk': 'are UK postcodes', + 'iban.mod97': 'are valid IBANs', + 'account.digits': 'look like account numbers', + 'ip.v4': 'are IPv4 addresses', + 'gender.values': 'are gender values', + 'money.amount': 'are money amounts', + 'expiry.shape': 'look like card expiry dates', + 'username.shape': 'look like usernames', + 'cred.password_hash': 'are password hashes', + 'cred.high_entropy': 'look like secrets', + 'cred.provider_key': 'are provider API keys', + 'cred.jwt': 'are JSON web tokens', + 'geo.precise': 'are precise coordinates', + 'age.range': 'are plausible ages', + 'card.luhn_iin': 'pass the card checksum', + 'card.shape': 'look like card numbers', + 'card.already_masked': 'are already-masked card numbers', + 'cvv.digits': 'look like card security codes', + 'passport.shape': 'look like passport numbers', + 'dl.shape': "look like driver's license numbers", + 'icd10.shape': 'are ICD-10 diagnosis codes', + 'blood_type.shape': 'are blood types', + 'ssn_us.dashed': 'are valid US SSNs', + 'ssn_us.digits': 'are 9-digit SSN candidates', + 'itin_us.dashed': 'are valid US ITINs', + dni_es: 'are valid Spanish DNIs', + cpf_br: 'are valid Brazilian CPFs', + cf_it: 'are valid Italian tax codes', + nir_fr: 'are valid French NIRs', + nino_uk: 'are valid UK NI numbers', + bsn_nl: 'are valid Dutch BSNs', + steuer_id_de: 'are valid German tax IDs', + pnr_se: 'are valid Swedish personal numbers', +}; + +const DETECTOR_NAMES: Record = { + 'email.syntax': 'Email syntax', + 'phone.formatted': 'Formatted phone', + 'phone.digits': 'Phone digits', + 'street.pattern': 'Street pattern', + 'postal.numeric': 'Numeric postal code', + 'postal.us_zip4': 'US ZIP+4', + 'postal.nl': 'Dutch postcode', + 'postal.uk': 'UK postcode', + 'iban.mod97': 'IBAN checksum', + 'account.digits': 'Account digits', + 'ip.v4': 'IPv4', + 'gender.values': 'Gender values', + 'money.amount': 'Money amount', + 'expiry.shape': 'Expiry date', + 'username.shape': 'Username shape', + 'cred.password_hash': 'Password hash', + 'cred.high_entropy': 'High-entropy secret', + 'cred.provider_key': 'Provider API key', + 'cred.jwt': 'JWT', + 'geo.precise': 'Precise coordinates', + 'age.range': 'Age range', + 'card.luhn_iin': 'Card checksum (Luhn)', + 'card.shape': 'Card shape', + 'cvv.digits': 'Security code digits', + 'passport.shape': 'Passport shape', + 'dl.shape': "Driver's license shape", + 'icd10.shape': 'ICD-10 code', + 'blood_type.shape': 'Blood type', + 'ssn_us.dashed': 'US SSN', + 'itin_us.dashed': 'US ITIN', + dni_es: 'Spanish DNI', + cpf_br: 'Brazilian CPF', + cf_it: 'Italian codice fiscale', + nir_fr: 'French NIR', + nino_uk: 'UK NI number', + bsn_nl: 'Dutch BSN', + steuer_id_de: 'German Steuer-ID', + pnr_se: 'Swedish personnummer', + 'name.lexicon': 'Name lexicon', + 'dob.plausible': 'Birth date plausibility', +}; + +const CLASS_LABEL: Record = { + national_id: 'National ID', + tax_id: 'Tax ID', + passport_number: 'Passport number', + drivers_license: "Driver's license", + card_number: 'Card number', + card_security: 'Card security code', + bank_account: 'Bank account', + credential: 'Credential', + special_category: 'Health & special category', + email: 'Email', + phone: 'Phone', + person_name: 'Person name', + street_address: 'Street address', + date_of_birth: 'Date of birth', + ip_address: 'IP address', + username: 'Username', + geo_precise: 'Precise location', + postal_code: 'Postal code', + gender: 'Gender', + age: 'Age', + card_expiry: 'Card expiry', + financial_personal: 'Personal financial', + free_text_pii: 'Free text with personal data', +}; + +const classFromRule = (rule: string) => rule.split('.')[0]; + +/** One prototype "why" line -> [points, kind, text], or null to drop. */ +export function humanize(why: string): Line | null { + let m: RegExpMatchArray | null; + if (why.startsWith('=')) return null; + + if ((m = why.match(/^([+-]\d+) name '([^']*)' -> .*?(strong|weak) phrase/))) { + const promoted = /promoted by person_table/.test(why); + return [ + Number(m[1]), + 'name', + `Column name "${m[2]}"${promoted ? ' in a person table' : ''}`, + ]; + } + if ((m = why.match(/^([+-]\d+) name '([^']*)' -> /))) { + return [Number(m[1]), 'name', `Column name "${m[2]}" matches no class`]; + } + if ( + (m = why.match( + /^([+-]\d+) values (\d+)\/(\d+) \(\d+%\) pass ([a-z0-9_.]+) \(\w+\)(.*)$/ + )) + ) { + const phrase = DETECTORS[m[4]] ?? `pass ${m[4]}`; + const few = /halved/.test(m[5]) ? ' (few distinct values)' : ''; + return [ + Number(m[1]), + 'values', + `${m[2]} of ${m[3]} values ${phrase}${few}`, + ]; + } + if ((m = why.match(/^([+-]\d+) values .*?, (\d+)% in the name lexicons/))) { + return [Number(m[1]), 'values', `${m[2]}% of values are known names`]; + } + if ( + (m = why.match( + /^([+-]\d+) values date-of-birth plausible: .*?(\d+) distinct years/ + )) + ) { + return [ + Number(m[1]), + 'values', + `Values look like birth dates (${m[2]} distinct years)`, + ]; + } + if ((m = why.match(/^([+-]\d+) values date-of-birth plausibility failed/))) { + return [Number(m[1]), 'values', "Values don't look like birth dates"]; + } + if ((m = why.match(/^([+-]\d+) values contradict the class/))) { + return [Number(m[1]), 'penalty', "Values don't fit the class"]; + } + if ((m = why.match(/^([+-]\d+) text validated PII spans in (\d+) rows/))) { + return [ + Number(m[1]), + 'text', + `Personal data inside the text in ${m[2]} rows`, + ]; + } + if ((m = why.match(/^([+-]\d+) text a few PII spans .*medium rows (\d+)/))) { + return [ + Number(m[1]), + 'text', + `Possible personal data inside the text in ${m[2]} rows`, + ]; + } + if ((m = why.match(/^\. type surrogate key/))) { + return [0, 'context', 'Key column, never classified']; + } + if ((m = why.match(/^\. sample only (\d+) non-null/))) { + return [0, 'context', `Only ${m[1]} non-null values sampled`]; + } + if ( + (m = why.match( + /^\. name phrase from ([a-z_]+)\.\S+ suppressed by negative "([^"]+)"/ + )) + ) { + const label = CLASS_LABEL[classFromRule(m[1])] ?? m[1]; + return [0, 'context', `"${m[2]}" rules out ${label.toLowerCase()}`]; + } + if ( + (m = why.match( + /^\. type ([a-z_]+)\.\S+ blocked by the type gate \(type:(\w+)\)/ + )) + ) { + const label = CLASS_LABEL[classFromRule(m[1])] ?? m[1]; + const article = /^[aeiou]/i.test(m[2]) ? 'An' : 'A'; + return [ + 0, + 'context', + `${article} ${m[2]} column can't hold ${label.toLowerCase()}`, + ]; + } + if (/^\. values value-only date_of_birth evidence ignored/.test(why)) { + return [0, 'context', 'The name says this is another kind of date']; + } + if ((m = why.match(/^([+-]\d+) (\w+) (.*)$/))) { + return [Number(m[1]), m[2] === 'text' ? 'text' : 'values', m[3]]; + } + return null; +} + +interface ProtoColumn { + verdict: 'enforce' | 'review' | 'clear'; + class: string | null; + tier: string | null; + confidence: number; + evidence: { + why?: string[]; + score?: { total: number }; + shapes?: [string, number][]; + type?: { db?: string }; + }; +} + +async function login(base: string, email: string, password: string) { + const res = await fetch(`${base}/api/v2/system/admin/session`, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ email, password }).toString(), + }); + if (!res.ok) throw new Error(`login ${res.status}`); + return ((await res.json()) as { session_token: string }).session_token; +} + +async function firstRows( + base: string, + token: string, + service: string, + table: string +): Promise[]> { + const url = `${base}/api/v2/${service}/_table/${encodeURIComponent(table)}?limit=50`; + const res = await fetch(url, { + headers: { 'X-DreamFactory-Session-Token': token }, + }); + if (!res.ok) return []; + return ( + ((await res.json()) as { resource?: Record[] }).resource ?? + [] + ); +} + +async function main() { + const base = process.env['DF_URL'] ?? 'http://127.0.0.1:8790'; + const token = await login( + base, + process.env['DF_ADMIN_EMAIL'] ?? 'admin@dmui.local', + process.env['DF_ADMIN_PASSWORD'] ?? '' + ); + const svcRes = await fetch( + `${base}/api/v2/system/service?fields=name,label`, + { + headers: { 'X-DreamFactory-Session-Token': token }, + } + ); + const labels = new Map( + ( + ((await svcRes.json()) as { resource: { name: string; label: string }[] }) + .resource ?? [] + ).map(s => [s.name, s.label]) + ); + + const inputs = process.argv.slice(2).map(a => a.split('=')); + const services: { service: string; label: string; tables: string[] }[] = []; + const columns: unknown[] = []; + let meta: Record = {}; + const detectorsByClass = new Map>(); + + for (const [service, file] of inputs) { + const proto = JSON.parse(readFileSync(file, 'utf8')) as { + meta: Record; + columns: Record; + }; + meta = proto.meta; + const byTable = new Map(); + for (const [key, col] of Object.entries(proto.columns)) { + const [, table, ...rest] = key.split('.'); + const column = rest.join('.'); + if (!byTable.has(table)) byTable.set(table, []); + byTable.get(table)!.push([column, col]); + for (const why of col.evidence.why ?? []) { + const d = why.match(/pass ([a-z0-9_.]+) \(\w+\)/)?.[1]; + if (col.class && d && DETECTOR_NAMES[d] && /^\+[1-9]/.test(why)) { + if (!detectorsByClass.has(col.class)) + detectorsByClass.set(col.class, new Set()); + detectorsByClass.get(col.class)!.add(DETECTOR_NAMES[d]); + } + } + } + + // Trim very wide demo databases: every table with a sensitive column, + // plus a handful of all-clear tables so the rail shows faint strips too. + let tables = [...byTable.keys()].sort((a, b) => a.localeCompare(b)); + const sensitive = tables.filter(t => + byTable.get(t)!.some(([, c]) => c.verdict !== 'clear') + ); + if (tables.length > 40) { + const clear = tables.filter(t => !sensitive.includes(t)).slice(0, 15); + tables = tables.filter(t => sensitive.includes(t) || clear.includes(t)); + } + services.push({ service, label: labels.get(service) ?? service, tables }); + + for (const table of tables) { + const cols = byTable.get(table)!; + const rows = cols.some(([, c]) => c.verdict !== 'clear') + ? await firstRows(base, token, service, table) + : []; + for (const [column, c] of cols) { + const verdict = + c.verdict === 'enforce' + ? 'masked' + : (c.verdict as 'review' | 'clear'); + let lines = (c.evidence.why ?? []) + .map(humanize) + .filter((l): l is Line => !!l); + if (verdict === 'clear') lines = lines.slice(0, 2); + const out: Record = { + id: `${service}.${table}.${column}`, + service, + table, + column, + type: c.evidence.type?.db ?? '', + verdict, + class: c.class, + tier: c.tier, + confidence: c.confidence, + total: c.evidence.score?.total ?? c.confidence, + lines, + }; + if (verdict !== 'clear' && c.evidence.shapes?.length) { + out['shapes'] = c.evidence.shapes.slice(0, 3); + } + const def = classDef(c.class); + if (verdict !== 'clear' && def) { + const present = rows + .map(r => r[column]) + .filter( + v => v !== null && v !== undefined && String(v).trim() !== '' + ) + .map(v => String(v)); + // Free text: prefer a row that actually has a span to mask. + // Free text: only a short prose row whose personal data was + // actually caught; anything else would leak text, so fall back + // to the synthetic sample. + const raw = + def.format === 'span' + ? present.find( + v => maskSpans(v) !== v && v.length < 90 && !/^[[{]/.test(v) + ) + : present[0]; + const value = raw === undefined ? def.sample : String(raw); + const examples: Record = {}; + for (const f of def.formats) { + examples[f] = applyMask(value.slice(0, 120), f, def.id); + } + out['examples'] = examples; + } + columns.push(out); + } + } + } + + const fixture = { + meta: { + ruleset: meta['ruleset'] ?? 'dreamfactory.core', + version: String(meta['ruleset_version'] ?? '').replace(/-spike.*$/, ''), + engine: meta['engine'], + generated: new Date().toISOString().slice(0, 10), + }, + services, + builtin: CLASSES.map(c => ({ + class: c.id, + detectors: [...(detectorsByClass.get(c.id) ?? [])].sort(), + })), + columns, + }; + const here = dirname(fileURLToPath(import.meta.url)); + const outFile = + process.env['DM_FIXTURE_OUT'] ?? + join( + here.includes('e2e') ? here : process.cwd(), + 'e2e/fixtures/data-masking/fixture.json' + ); + writeFileSync(outFile, JSON.stringify(fixture)); + console.log(`wrote ${outFile}: ${columns.length} columns`); +} + +main().catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/jest.config.ci.js b/jest.config.ci.js index 302ad8139..ee7de3b23 100644 --- a/jest.config.ci.js +++ b/jest.config.ci.js @@ -66,5 +66,22 @@ module.exports = { '/src/app/shared/components/df-secret/df-secret.component.spec.ts', // 2026-10-05 searchable pickers (#534): dfSearch filter + componentTypes. '/src/app/shared/components/df-select-search/df-select-search.spec.ts', + // 2026-10-29 Data Masking: mask formats, API service, store (polling, + // write ordering, load failures), masked-value privacy guard, shell + // mode confirm, Columns review flow. + '/src/app/adf-data-masking/**/*.spec.ts', + // 2026-10-29 Compliance baselines: API service (report as text), store + // (selection, apply, stop tracking, export), page (apply confirm lists + // the server's changes, fix links, manual checks). + '/src/app/adf-compliance/**/*.spec.ts', + // 2026-10-29 Data Masking on shared/existing screens: the standard + // paywall without the package (every license), palette keywords, + // Security overview status, Data Explorer masking marker and + // ignored-filter note. + '/src/app/shared/services/df-paywall.service.spec.ts', + '/src/app/shared/services/action-registry.service.spec.ts', + '/src/app/adf-section-overviews/df-security-overview.component.spec.ts', + '/src/app/adf-data-explorer/masking-notice.spec.ts', + '/src/app/adf-data-explorer/df-data-grid.masking.spec.ts', ], }; diff --git a/src/app/adf-compliance/compliance-api.service.spec.ts b/src/app/adf-compliance/compliance-api.service.spec.ts new file mode 100644 index 000000000..fbe32b2e8 --- /dev/null +++ b/src/app/adf-compliance/compliance-api.service.spec.ts @@ -0,0 +1,96 @@ +import { TestBed } from '@angular/core/testing'; +import { + HttpClientTestingModule, + HttpTestingController, +} from '@angular/common/http/testing'; +import { + DfComplianceApiService, + SECURITY_BASELINE_URL, +} from './compliance-api.service'; +import { ERROR_HANDLING } from 'src/app/shared/utilities/http-contexts'; + +describe('DfComplianceApiService', () => { + let api: DfComplianceApiService; + let http: HttpTestingController; + + beforeEach(() => { + TestBed.configureTestingModule({ imports: [HttpClientTestingModule] }); + api = TestBed.inject(DfComplianceApiService); + http = TestBed.inject(HttpTestingController); + }); + + afterEach(() => http.verify()); + + it('lists baselines from the system resource and unwraps resource', done => { + expect(SECURITY_BASELINE_URL).toBe('/api/v2/system/security_baseline'); + api.list().subscribe(list => { + expect(list.map(b => b.id)).toEqual(['hipaa']); + done(); + }); + const req = http.expectOne(SECURITY_BASELINE_URL); + expect(req.request.method).toBe('GET'); + req.flush({ resource: [{ id: 'hipaa' }] }); + }); + + it('reads one baseline by (encoded) id', () => { + api.get('pci_dss').subscribe(); + http.expectOne(`${SECURITY_BASELINE_URL}/pci_dss`).flush({}); + api.get('a b').subscribe(); + http.expectOne(`${SECURITY_BASELINE_URL}/a%20b`).flush({}); + }); + + it('reads what Apply with masking on would change, quietly', done => { + api.changesWithModeOn('hipaa').subscribe(changes => { + expect(changes).toEqual([{ id: 'mode', text: 'Turn masking on.' }]); + done(); + }); + const req = http.expectOne( + r => r.url === `${SECURITY_BASELINE_URL}/hipaa` && r.method === 'GET' + ); + expect(req.request.params.get('mode')).toBe('on'); + expect(req.request.context.get(ERROR_HANDLING)).toBe('silent'); + req.flush({ changes: [{ id: 'mode', text: 'Turn masking on.' }] }); + }); + + it('applies with an empty body, or mode on when asked; toast-off', () => { + api.apply('hipaa').subscribe(); + let req = http.expectOne(`${SECURITY_BASELINE_URL}/hipaa/apply`); + expect(req.request.method).toBe('POST'); + expect(req.request.body).toEqual({}); + expect(req.request.context.get(ERROR_HANDLING)).toBe('toast-off'); + req.flush({}); + api.apply('hipaa', true).subscribe(); + req = http.expectOne(`${SECURITY_BASELINE_URL}/hipaa/apply`); + expect(req.request.body).toEqual({ mode: 'on' }); + req.flush({}); + }); + + it('stops tracking with DELETE', () => { + api.stopTracking('gdpr').subscribe(); + const req = http.expectOne(`${SECURITY_BASELINE_URL}/gdpr`); + expect(req.request.method).toBe('DELETE'); + expect(req.request.context.get(ERROR_HANDLING)).toBe('toast-off'); + req.flush({}); + }); + + it.each([ + ['json', 'application/json', '{"baseline":{"id":"soc2"}}'], + ['csv', 'text/csv', 'control,status\nmasking.mode,pass\n'], + ] as const)( + 'downloads the %s report verbatim as text', + (format, accept, body) => { + let got: string | undefined; + api.report('soc2', format).subscribe(text => (got = text)); + const req = http.expectOne( + r => r.url === `${SECURITY_BASELINE_URL}/soc2/report` + ); + expect(req.request.params.get('format')).toBe(format); + expect(req.request.responseType).toBe('text'); + expect(req.request.headers.get('Accept')).toBe(accept); + expect(req.request.context.get(ERROR_HANDLING)).toBe('toast-off'); + req.flush(body); + // Never camelCased or parsed: what the server sent is what is saved. + expect(got).toBe(body); + } + ); +}); diff --git a/src/app/adf-compliance/compliance-api.service.ts b/src/app/adf-compliance/compliance-api.service.ts new file mode 100644 index 000000000..957b0e2d9 --- /dev/null +++ b/src/app/adf-compliance/compliance-api.service.ts @@ -0,0 +1,86 @@ +/** + * HTTP for /api/v2/system/security_baseline (backend contract, section 8). + * Reads use the default error handling (normalized + toasted by the error + * interceptor); writes and report downloads are toast-off, so the store + * shows exactly one toast and keeps what it had. + */ +import { HttpClient, HttpHeaders } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Observable, map } from 'rxjs'; +import { BASE_URL } from 'src/app/shared/constants/urls'; +import { silent, toastOff } from 'src/app/shared/utilities/http-contexts'; +import { + BaselineChange, + BaselineDetail, + BaselineSummary, + ReportFormat, +} from './compliance.types'; + +export const SECURITY_BASELINE_URL = `${BASE_URL}/system/security_baseline`; + +const REPORT_ACCEPT: Record = { + json: 'application/json', + csv: 'text/csv', +}; + +@Injectable({ providedIn: 'root' }) +export class DfComplianceApiService { + constructor(private http: HttpClient) {} + + private url(id: string, sub = ''): string { + return `${SECURITY_BASELINE_URL}/${encodeURIComponent(id)}${sub}`; + } + + list(): Observable { + return this.http + .get<{ resource: BaselineSummary[] }>(SECURITY_BASELINE_URL) + .pipe(map(r => r?.resource ?? [])); + } + + get(id: string): Observable { + return this.http.get(this.url(id)); + } + + /** + * What Apply with `{mode: 'on'}` would change now (GET ?mode=on): the + * same list as `changes`, plus turning masking on. Read-only, and quiet: + * the Apply dialog falls back to the plain list if it fails. + */ + changesWithModeOn(id: string): Observable { + return this.http + .get(this.url(id), { + params: { mode: 'on' }, + context: silent(), + }) + .pipe(map(d => d?.changes ?? [])); + } + + /** Adopts the baseline and applies its changes; `turnOn` also sets mode on. */ + apply(id: string, turnOn = false): Observable { + return this.http.post( + this.url(id, '/apply'), + turnOn ? { mode: 'on' } : {}, + { context: toastOff() } + ); + } + + /** Stop tracking: the baseline is no longer adopted; settings stay. */ + stopTracking(id: string): Observable { + return this.http.delete(this.url(id), { context: toastOff() }); + } + + /** + * The evidence report, verbatim. Read as text, not as a blob: the case + * interceptor camelCases every application/json body it sees, and a Blob + * run through it comes back as `{}`. Text passes through untouched, so + * the saved file is byte-for-byte what the server sent. + */ + report(id: string, format: ReportFormat): Observable { + return this.http.get(this.url(id, '/report'), { + params: { format }, + headers: new HttpHeaders({ Accept: REPORT_ACCEPT[format] }), + responseType: 'text', + context: toastOff(), + }); + } +} diff --git a/src/app/adf-compliance/compliance-testing.ts b/src/app/adf-compliance/compliance-testing.ts new file mode 100644 index 000000000..5f3c788da --- /dev/null +++ b/src/app/adf-compliance/compliance-testing.ts @@ -0,0 +1,115 @@ +/** Shared Jest helpers for the Compliance specs. */ +import { Translation, TranslocoTestingModule } from '@ngneat/transloco'; +import { MaskingOverview } from 'src/app/adf-data-masking/data-masking.types'; +import { + BaselineControl, + BaselineDetail, + BaselineSummary, +} from './compliance.types'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const CP: Translation = require('../../assets/i18n/compliance/en.json'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const DM: Translation = require('../../assets/i18n/dataMasking/en.json'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const ROOT: Translation = require('../../assets/i18n/en.json'); + +export function cpTransloco() { + return TranslocoTestingModule.forRoot({ + langs: { en: { ...ROOT, compliance: CP, dataMasking: DM } }, + translocoConfig: { availableLangs: ['en'], defaultLang: 'en' }, + preloadLangs: true, + }); +} + +export function summary(over: Partial = {}): BaselineSummary { + return { + id: 'hipaa', + name: 'HIPAA', + vertical: 'healthcare', + frameworks: ['HIPAA Security Rule'], + adopted: false, + score: { pass: 9, fail: 3, manual: 2, total: 14 }, + ...over, + }; +} + +export function control(over: Partial = {}): BaselineControl { + return { + id: 'masking.mode', + title: 'Masking is on', + refs: ['45 CFR 164.312(a)(1)'], + status: 'pass', + detail: null, + fix: { kind: 'none', route: null }, + ...over, + }; +} + +export function detail(over: Partial = {}): BaselineDetail { + return { + id: 'hipaa', + name: 'HIPAA', + vertical: 'healthcare', + frameworks: ['HIPAA Security Rule'], + adopted: false, + adoptedAt: null, + appliedAt: null, + controls: [ + control(), + control({ + id: 'masking.review', + title: 'Review queue is empty', + status: 'fail', + detail: '24 columns to review', + refs: ['45 CFR 164.514(b)'], + fix: { + kind: 'link', + route: '/api-security/data-masking?tab=columns', + }, + }), + control({ + id: 'access.ai', + title: 'AI sessions always see masked values', + status: 'fail', + refs: ['45 CFR 164.312(a)(1)'], + fix: { kind: 'apply', route: null }, + }), + control({ + id: 'platform.rate_limit', + title: 'Rate limits are active', + status: 'na', + refs: [], + }), + control({ + id: 'manual.encryption', + title: 'Encryption at rest on your databases', + status: 'manual', + refs: ['45 CFR 164.312(a)(2)(iv)'], + }), + ], + changes: [ + { id: 'access.ai', text: 'AI and MCP sessions: always masked' }, + { id: 'masking.mode', text: 'Masking mode: Preview' }, + ], + ...over, + }; +} + +/** A masking overview as the store reads it (scanned unless said). */ +export function overview( + mode: 'off' | 'preview' | 'on', + scan: Record = {} +): MaskingOverview { + return { + mode, + scan: { + status: 'idle', + finishedAt: '2026-10-28T10:00:00Z', + columns: 40, + ...scan, + }, + counts: { masked: 10, review: 2, clear: 28 }, + you: { canGrant: true, seesRealValues: false }, + } as MaskingOverview; +} diff --git a/src/app/adf-compliance/compliance.catalog.ts b/src/app/adf-compliance/compliance.catalog.ts new file mode 100644 index 000000000..d341264c1 --- /dev/null +++ b/src/app/adf-compliance/compliance.catalog.ts @@ -0,0 +1,45 @@ +/** + * Compliance baselines catalog: the fixed verticals, their order, icons and + * the baseline each one opens by default. Baseline ids are fixed by the + * backend contract (hipaa, pci_dss, glba, cjis, gdpr, soc2, iso27001), so + * the first-run row on Data Masking can link straight to one without a + * request. Pure TypeScript: shared with the Playwright mock. + * + * Labels live in i18n (`verticals.`), this file only holds ids. + */ + +export type Vertical = + | 'healthcare' + | 'payments' + | 'banking' + | 'government' + | 'privacy' + | 'general'; + +export interface VerticalDef { + id: Vertical; + /** Material icon ligature. */ + icon: string; + /** Baseline the vertical opens on when chosen from a shortcut. */ + baseline: string; +} + +export const VERTICALS: VerticalDef[] = [ + { id: 'healthcare', icon: 'health_and_safety', baseline: 'hipaa' }, + { id: 'payments', icon: 'credit_card', baseline: 'pci_dss' }, + { id: 'banking', icon: 'account_balance', baseline: 'glba' }, + { id: 'government', icon: 'gavel', baseline: 'cjis' }, + { id: 'privacy', icon: 'privacy_tip', baseline: 'gdpr' }, + { id: 'general', icon: 'verified_user', baseline: 'soc2' }, +]; + +const ORDER = new Map(VERTICALS.map((v, i) => [v.id as string, i])); + +/** Sort position of a vertical; unknown verticals go last, in order seen. */ +export function verticalOrder(id: string): number { + return ORDER.get(id) ?? VERTICALS.length; +} + +export function verticalDef(id: string): VerticalDef | undefined { + return VERTICALS.find(v => v.id === id); +} diff --git a/src/app/adf-compliance/compliance.resolver.ts b/src/app/adf-compliance/compliance.resolver.ts new file mode 100644 index 000000000..a0fe7e798 --- /dev/null +++ b/src/app/adf-compliance/compliance.resolver.ts @@ -0,0 +1,14 @@ +import { inject } from '@angular/core'; +import { ResolveFn } from '@angular/router'; +import { map } from 'rxjs'; +import { DfPaywallService } from 'src/app/shared/services/df-paywall.service'; + +/** + * 'paywall' when this instance has no security_baseline system resource + * (the df-masking package is not there), on any license, like Rate + * Limiting, Scheduler and Reporting; otherwise 'ok'. + */ +export const complianceResolver: ResolveFn<'paywall' | 'ok'> = () => + inject(DfPaywallService) + .activatePaywall('security_baseline') + .pipe(map(missing => (missing ? 'paywall' : 'ok'))); diff --git a/src/app/adf-compliance/compliance.store.spec.ts b/src/app/adf-compliance/compliance.store.spec.ts new file mode 100644 index 000000000..ee340633a --- /dev/null +++ b/src/app/adf-compliance/compliance.store.spec.ts @@ -0,0 +1,572 @@ +import { + TestBed, + discardPeriodicTasks, + fakeAsync, + tick, +} from '@angular/core/testing'; +import { + HttpClient, + HttpErrorResponse, + provideHttpClient, + withInterceptors, +} from '@angular/common/http'; +import { + HttpTestingController, + provideHttpClientTesting, +} from '@angular/common/http/testing'; +import { provideRouter } from '@angular/router'; +import { caseInterceptor } from 'src/app/shared/interceptors/case.interceptor'; +import { errorInterceptor } from 'src/app/shared/interceptors/error.interceptor'; +import { DfUserDataService } from 'src/app/shared/services/df-user-data.service'; +import { DfTrialService } from 'src/app/shared/services/df-trial.service'; +import { normalizeError } from 'src/app/shared/utilities/app-error'; +import { Subject, of, throwError } from 'rxjs'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { DfDataMaskingApiService } from 'src/app/adf-data-masking/data-masking-api.service'; +import * as fileUtils from 'src/app/shared/utilities/file'; +import { DfComplianceApiService } from './compliance-api.service'; +import { + ComplianceStore, + SCAN_POLL_MS, + fixLink, + parseTextError, + scoreOf, +} from './compliance.store'; +import { + control, + cpTransloco, + detail, + overview, + summary, +} from './compliance-testing'; +import { BaselineDetail } from './compliance.types'; + +jest.mock('src/app/shared/utilities/file', () => ({ + saveRawAsFile: jest.fn(), +})); + +describe('ComplianceStore', () => { + let store: ComplianceStore; + let api: Record; + let masking: { overview: jest.Mock }; + let snack: { openSnackBar: jest.Mock; openError: jest.Mock }; + + const list = () => [ + summary({ id: 'soc2', name: 'SOC 2', vertical: 'general' }), + summary(), + summary({ + id: 'pci_dss', + name: 'PCI DSS', + vertical: 'payments', + adopted: true, + }), + summary({ id: 'iso27001', name: 'ISO 27001', vertical: 'general' }), + ]; + + beforeEach(() => { + api = { + list: jest.fn(() => of(list())), + get: jest.fn((id: string) => of(detail({ id, name: id.toUpperCase() }))), + apply: jest.fn(), + stopTracking: jest.fn(() => of({})), + report: jest.fn(() => of('{"ok":true}')), + }; + masking = { + overview: jest.fn(() => of(overview('preview'))), + }; + snack = { openSnackBar: jest.fn(), openError: jest.fn() }; + (fileUtils.saveRawAsFile as jest.Mock).mockClear(); + TestBed.configureTestingModule({ + imports: [cpTransloco()], + providers: [ + ComplianceStore, + { provide: DfComplianceApiService, useValue: api }, + { provide: DfDataMaskingApiService, useValue: masking }, + { provide: DfSnackbarService, useValue: snack }, + ], + }); + store = TestBed.inject(ComplianceStore); + }); + + describe('load and select', () => { + it('selects the requested baseline when it exists', () => { + store.load('soc2'); + expect(store.selectedId).toBe('soc2'); + expect(api['get']).toHaveBeenCalledWith('soc2'); + expect(store.maskingMode).toBe('preview'); + // Only a peek for the turn-on option: never toasts on failure. + expect(masking.overview).toHaveBeenCalledWith(true); + }); + + it('otherwise the first tracked one, else the first', () => { + store.load('nope'); + expect(store.selectedId).toBe('pci_dss'); + api['list'].mockReturnValue(of([summary({ id: 'gdpr' })])); + store.load(null); + expect(store.selectedId).toBe('gdpr'); + }); + + it('groups by vertical in catalog order', () => { + store.load(); + expect(store.groups.map(g => g.vertical)).toEqual([ + 'healthcare', + 'payments', + 'general', + ]); + expect(store.groups[2].baselines.map(b => b.id)).toEqual([ + 'soc2', + 'iso27001', + ]); + expect(store.groups[0].icon).toBe('health_and_safety'); + }); + + it('marks a failed list so the page can offer retry', () => { + api['list'].mockReturnValue(throwError(() => ({ status: 500 }))); + store.load(); + expect(store.loadFailed).toBe(true); + expect(store.loading).toBe(false); + expect(store.loadError).toBeNull(); + }); + + it("keeps the server's reason when the list fails with one", () => { + const notSetUp = normalizeError( + new HttpErrorResponse({ + status: 503, + error: { + error: { + code: 503, + message: + 'Data Masking is not set up yet. Run: php artisan df:masking setup', + }, + }, + }) + ); + api['list'].mockReturnValue(throwError(() => notSetUp)); + store.load(); + expect(store.loadError).toBe( + 'Data Masking is not set up yet. Run: php artisan df:masking setup' + ); + }); + + it('a slow detail for a baseline already left never fails the new one', () => { + store.load('soc2'); + const slow = new Subject(); + api['get'] + .mockReturnValueOnce(slow) + .mockReturnValueOnce(of(detail({ id: 'hipaa' }))); + store.select('iso27001'); + store.select('hipaa'); + slow.error({ status: 500 }); + expect(store.detailFailed).toBe(false); + expect(store.detail?.id).toBe('hipaa'); + }); + + it('keeps the rail score in step with a fresh detail', () => { + store.load('hipaa'); + const hipaa = store.baselines!.find(b => b.id === 'hipaa')!; + // detail(): 1 pass, 2 fail, 1 manual, 1 n/a + expect(hipaa.score.pass).toBe(1); + expect(hipaa.score.fail).toBe(2); + expect(hipaa.score.manual).toBe(1); + }); + }); + + describe('apply', () => { + beforeEach(() => store.load('hipaa')); + + it('adopts, refreshes every score, and says so', () => { + store.details.set('soc2', detail({ id: 'soc2' })); + api['apply'].mockReturnValue(of(detail({ adopted: true, changes: [] }))); + store.apply('hipaa', false).subscribe(); + expect(api['apply']).toHaveBeenCalledWith('hipaa', false); + // Settings are instance-wide: other cached checks are stale. + expect(store.details.has('soc2')).toBe(false); + expect(store.detail?.adopted).toBe(true); + expect(api['list']).toHaveBeenCalledTimes(2); + expect(masking.overview).toHaveBeenCalledTimes(2); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'HIPAA baseline applied', + 'success' + ); + expect(store.busy).toBeNull(); + }); + + it('says masking is on when it was turned on too', () => { + api['apply'].mockReturnValue(of(detail({ adopted: true }))); + store.apply('hipaa', true).subscribe(); + expect(api['apply']).toHaveBeenCalledWith('hipaa', true); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'HIPAA baseline applied. Masking is on.', + 'success' + ); + }); + + it('on a never-scanned instance says the scan started, polls it, then re-reads the checks', fakeAsync(() => { + masking.overview.mockReturnValue( + of(overview('preview', { finishedAt: null, columns: 0 })) + ); + store.load('hipaa'); + expect(store.neverScanned).toBe(true); + expect(store.offerTurnOn).toBe(false); + store.details.set('soc2', detail({ id: 'soc2' })); + api['apply'].mockReturnValue( + of({ ...detail({ adopted: true, changes: [] }), scanStarted: true }) + ); + // The server just started the scan. + masking.overview.mockReturnValue( + of( + overview('preview', { + status: 'running', + progress: 40, + finishedAt: null, + }) + ) + ); + store.apply('hipaa', false).subscribe(); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'HIPAA baseline applied. Scanning…', + 'success' + ); + expect(store.scanning).toBe(true); + expect(store.scanProgress).toBe(40); + expect(store.offerTurnOn).toBe(false); + api['get'].mockClear(); + api['list'].mockClear(); + tick(SCAN_POLL_MS); + expect(store.scanning).toBe(true); + expect(api['get']).not.toHaveBeenCalled(); + // The scan finishes: the checks are read again. + masking.overview.mockReturnValue(of(overview('preview'))); + tick(SCAN_POLL_MS); + expect(store.scanning).toBe(false); + expect(store.neverScanned).toBe(false); + expect(store.offerTurnOn).toBe(true); + expect(api['get']).toHaveBeenCalledWith('hipaa'); + expect(api['list']).toHaveBeenCalledTimes(1); + const calls = masking.overview.mock.calls.length; + tick(SCAN_POLL_MS * 3); + expect(masking.overview.mock.calls.length).toBe(calls); + discardPeriodicTasks(); + })); + + it('shows the scan Apply started even before the overview reports it running', fakeAsync(() => { + masking.overview.mockReturnValue( + of(overview('preview', { finishedAt: null, columns: 0 })) + ); + store.load('hipaa'); + api['apply'].mockReturnValue( + of({ ...detail({ adopted: true, changes: [] }), scanStarted: true }) + ); + store.apply('hipaa', false).subscribe(); + // The overview still says idle and never scanned: the scan is queued. + expect(store.scanning).toBe(true); + api['get'].mockClear(); + masking.overview.mockReturnValue(of(overview('preview'))); + tick(SCAN_POLL_MS); + expect(store.scanning).toBe(false); + expect(api['get']).toHaveBeenCalledWith('hipaa'); + discardPeriodicTasks(); + })); + + it('keeps what it had and toasts once on failure', () => { + const before = store.detail; + api['apply'].mockReturnValue(throwError(() => ({ status: 403 }))); + store.apply('hipaa', false).subscribe({ error: () => undefined }); + expect(store.detail).toBe(before); + expect(snack.openError).toHaveBeenCalledTimes(1); + expect(store.busy).toBeNull(); + }); + }); + + describe('turning masking on', () => { + it('is offered once a scan has finished and masking is not on', () => { + store.load('hipaa'); + expect(store.offerTurnOn).toBe(true); + masking.overview.mockReturnValue(of(overview('on'))); + store.load('hipaa'); + expect(store.offerTurnOn).toBe(false); + }); + + it('is not offered after a scan that found no columns (the server refuses On)', () => { + masking.overview.mockReturnValue( + of({ + ...overview('off', { columns: 0 }), + counts: { masked: 0, review: 0, clear: 0 }, + }) + ); + store.load('hipaa'); + expect(store.neverScanned).toBe(true); + expect(store.offerTurnOn).toBe(false); + }); + + it('is never offered before the first scan, or when masking is unknown', () => { + masking.overview.mockReturnValue( + of(overview('off', { finishedAt: null, columns: 0 })) + ); + store.load('hipaa'); + expect(store.neverScanned).toBe(true); + expect(store.offerTurnOn).toBe(false); + masking.overview.mockReturnValue(throwError(() => ({ status: 503 }))); + store.load('hipaa'); + expect(store.offerTurnOn).toBe(false); + }); + + it('shows a scan already running when the page opens, and stops polling on destroy', fakeAsync(() => { + masking.overview.mockReturnValue( + of(overview('preview', { status: 'running', progress: null })) + ); + store.load('hipaa'); + expect(store.scanning).toBe(true); + expect(store.scanProgress).toBeNull(); + store.ngOnDestroy(); + const calls = masking.overview.mock.calls.length; + tick(SCAN_POLL_MS * 2); + expect(masking.overview.mock.calls.length).toBe(calls); + })); + }); + + it('stop tracking clears the tracked state and reloads the checks', () => { + api['get'].mockImplementation((id: string) => + of(detail({ id, name: 'PCI DSS', adopted: true })) + ); + store.load('pci_dss'); + expect(store.selected?.adopted).toBe(true); + api['get'].mockReset(); + api['get'].mockReturnValue(new Subject()); + store.stopTracking('pci_dss'); + expect(api['stopTracking']).toHaveBeenCalledWith('pci_dss'); + expect(store.baselines!.find(b => b.id === 'pci_dss')!.adopted).toBe(false); + expect(api['get']).toHaveBeenCalledWith('pci_dss'); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'Stopped tracking PCI DSS', + 'success' + ); + }); + + describe('export', () => { + it('saves the report exactly as served, named by baseline and day', () => { + store.exportReport('hipaa', 'json'); + const save = fileUtils.saveRawAsFile as jest.Mock; + expect(api['report']).toHaveBeenCalledWith('hipaa', 'json'); + expect(save).toHaveBeenCalledTimes(1); + const [text, name, type] = save.mock.calls[0]; + expect(text).toBe('{"ok":true}'); + expect(name).toMatch( + /^dreamfactory-hipaa-baseline-\d{4}-\d{2}-\d{2}\.json$/ + ); + expect(type).toBe('json'); + expect(store.busy).toBeNull(); + }); + + it('surfaces the server message from a text error body', () => { + api['report'].mockReturnValue( + throwError( + () => + new HttpErrorResponse({ + status: 403, + error: JSON.stringify({ + error: { code: 403, message: 'Not allowed' }, + }), + }) + ) + ); + store.exportReport('hipaa', 'csv'); + expect(snack.openError).toHaveBeenCalledWith( + expect.objectContaining({ status: 403, message: 'Not allowed' }) + ); + expect(fileUtils.saveRawAsFile).not.toHaveBeenCalled(); + }); + }); + + describe('helpers', () => { + it('scores pass and fail only', () => { + expect(scoreOf(detail().controls)).toEqual({ pass: 1, checked: 3 }); + expect(scoreOf([control({ status: 'manual' })])).toEqual({ + pass: 0, + checked: 0, + }); + }); + + it('turns a fix route into router commands, in-app only', () => { + expect(fixLink('/api-security/data-masking?tab=access')).toEqual({ + path: '/api-security/data-masking', + query: { tab: 'access' }, + }); + expect(fixLink('api-security/rate-limiting')).toEqual({ + path: '/api-security/rate-limiting', + query: {}, + }); + expect(fixLink('#/system-settings/cors')).toEqual({ + path: '/system-settings/cors', + query: {}, + }); + expect(fixLink('/dreamfactory/dist/#/admin-settings/admins')).toEqual({ + path: '/admin-settings/admins', + query: {}, + }); + expect(fixLink('https://evil.example/x')).toBeNull(); + expect(fixLink('javascript:alert(1)')).toBeNull(); + expect(fixLink('//evil.example')).toBeNull(); + expect(fixLink('')).toBeNull(); + expect(fixLink(null)).toBeNull(); + expect(fixLink('/')).toBeNull(); + }); + + it('leaves non-text errors alone', () => { + const e = { status: 500 }; + expect(parseTextError(e)).toBe(e); + const bad = new HttpErrorResponse({ status: 502, error: '' }); + expect(parseTextError(bad)).toBe(bad); + const html = normalizeError(bad); + expect(parseTextError(html)).toBe(html); + }); + + it('reads the envelope out of an already-normalized text error', () => { + const appError = normalizeError( + new HttpErrorResponse({ + status: 404, + url: '/api/v2/system/security_baseline/x/report', + error: JSON.stringify({ + error: { code: 404, message: 'Baseline not found' }, + }), + }) + ); + expect(appError.message).toBe('errors.http4xx'); + expect(parseTextError(appError)).toEqual( + expect.objectContaining({ status: 404, message: 'Baseline not found' }) + ); + }); + }); +}); + +// -------------------------------------------------------------------------- +// Report download errors through the app's real interceptor chain (RU-10). +// -------------------------------------------------------------------------- +describe('ComplianceStore export errors (real interceptors)', () => { + it("shows the server's message when the evidence report fails", () => { + const snack = { openSnackBar: jest.fn(), openError: jest.fn() }; + TestBed.resetTestingModule(); + TestBed.configureTestingModule({ + imports: [cpTransloco()], + providers: [ + provideRouter([]), + provideHttpClient( + withInterceptors([caseInterceptor, errorInterceptor]) + ), + provideHttpClientTesting(), + ComplianceStore, + DfComplianceApiService, + { provide: DfSnackbarService, useValue: snack }, + { + provide: DfUserDataService, + useValue: { clearToken: jest.fn(), userData: null }, + }, + { provide: DfTrialService, useValue: { markLocked: jest.fn() } }, + { + provide: DfDataMaskingApiService, + useValue: { overview: jest.fn(() => of(overview('preview'))) }, + }, + ], + }); + TestBed.inject(HttpClient); + const http = TestBed.inject(HttpTestingController); + const store = TestBed.inject(ComplianceStore); + store.exportReport('hipaa', 'csv'); + const req = http.expectOne(r => + r.url.includes('/security_baseline/hipaa/report') + ); + // DreamFactory's JSON error envelope; responseType 'text' hands it to + // the app as a string, and the error interceptor normalizes it first. + req.flush( + JSON.stringify({ + error: { code: 403, message: 'Not allowed to export this baseline' }, + }), + { status: 403, statusText: 'Forbidden' } + ); + expect(snack.openError).toHaveBeenCalledTimes(1); + expect(snack.openError.mock.calls[0][0].message).toBe( + 'Not allowed to export this baseline' + ); + expect(fileUtils.saveRawAsFile).not.toHaveBeenCalled(); + http.verify(); + }); +}); + +describe('ComplianceStore writes refused with 403 (real interceptors)', () => { + const MESSAGE = + 'Only a system administrator can change Compliance baselines.'; + let store: ComplianceStore; + let http: HttpTestingController; + let snack: { openSnackBar: jest.Mock; openError: jest.Mock }; + let user: { clearToken: jest.Mock; userData: null }; + + beforeEach(() => { + snack = { openSnackBar: jest.fn(), openError: jest.fn() }; + user = { clearToken: jest.fn(), userData: null }; + TestBed.resetTestingModule(); + TestBed.configureTestingModule({ + imports: [cpTransloco()], + providers: [ + provideRouter([]), + provideHttpClient( + withInterceptors([caseInterceptor, errorInterceptor]) + ), + provideHttpClientTesting(), + ComplianceStore, + DfComplianceApiService, + { provide: DfSnackbarService, useValue: snack }, + { provide: DfUserDataService, useValue: user }, + { provide: DfTrialService, useValue: { markLocked: jest.fn() } }, + { + provide: DfDataMaskingApiService, + useValue: { overview: jest.fn(() => of(overview('preview'))) }, + }, + ], + }); + TestBed.inject(HttpClient); + http = TestBed.inject(HttpTestingController); + store = TestBed.inject(ComplianceStore); + store.load('hipaa'); + http + .expectOne(r => r.url.endsWith('/security_baseline')) + .flush({ resource: [{ ...summary(), adopted: true }] }); + http + .expectOne(r => r.url.endsWith('/security_baseline/hipaa')) + .flush({ ...detail(), adopted: true }); + }); + + afterEach(() => http.verify()); + + const forbid = (method: string) => + http + .expectOne(r => r.method === method) + .flush( + { error: { code: 403, message: MESSAGE, status_code: 403 } }, + { status: 403, statusText: 'Forbidden' } + ); + + const expectServerToast = () => { + expect(snack.openError).toHaveBeenCalledTimes(1); + expect(snack.openError.mock.calls[0][0].message).toBe(MESSAGE); + expect(user.clearToken).not.toHaveBeenCalled(); + expect(snack.openSnackBar).not.toHaveBeenCalled(); + }; + + it('apply: the checks and tracking stay as the server has them', () => { + const before = store.detail; + store.apply('hipaa', true).subscribe({ error: () => undefined }); + forbid('POST'); + expect(store.detail).toBe(before); + expect(store.busy).toBeNull(); + expectServerToast(); + }); + + it('stop tracking: still tracked', () => { + store.stopTracking('hipaa'); + forbid('DELETE'); + expect(store.selected?.adopted).toBe(true); + expect(store.baselines?.[0].adopted).toBe(true); + expect(store.busy).toBeNull(); + expectServerToast(); + }); +}); diff --git a/src/app/adf-compliance/compliance.store.ts b/src/app/adf-compliance/compliance.store.ts new file mode 100644 index 000000000..b4adb3b49 --- /dev/null +++ b/src/app/adf-compliance/compliance.store.ts @@ -0,0 +1,462 @@ +/** + * State for the Compliance page. The page provides one instance. Baselines + * load as a list; a baseline's checks load on selection and are cached. + * Apply changes instance-wide settings, so after it every other baseline's + * score is stale: the list reloads and the other cached details drop. + */ +import { HttpErrorResponse } from '@angular/common/http'; +import { Injectable, OnDestroy, inject } from '@angular/core'; +import { TranslocoService } from '@ngneat/transloco'; +import { + Observable, + Subscription, + catchError, + finalize, + of, + switchMap, + tap, + timer, +} from 'rxjs'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { isAppError, normalizeError } from 'src/app/shared/utilities/app-error'; +import { saveRawAsFile } from 'src/app/shared/utilities/file'; +import { DfDataMaskingApiService } from 'src/app/adf-data-masking/data-masking-api.service'; +import { + MaskingMode, + MaskingOverview, +} from 'src/app/adf-data-masking/data-masking.types'; +import { DfComplianceApiService } from './compliance-api.service'; +import { VERTICALS, verticalOrder } from './compliance.catalog'; +import { + BaselineChange, + BaselineControl, + BaselineDetail, + BaselineSummary, + ReportFormat, +} from './compliance.types'; + +export interface VerticalGroup { + vertical: string; + icon: string; + baselines: BaselineSummary[]; +} + +/** Automated checks only: manual and n/a never count for or against. */ +export function scoreOf(controls: BaselineControl[]): { + pass: number; + checked: number; +} { + let pass = 0; + let fail = 0; + for (const c of controls) { + if (c.status === 'pass') pass++; + else if (c.status === 'fail') fail++; + } + return { pass, checked: pass + fail }; +} + +/** + * Where a control's Fix link goes, as router commands. Only in-app routes + * are honored ("/api-security/data-masking?tab=access", with or without the + * leading slash or the "#"); anything with a scheme is dropped. + */ +export function fixLink( + route: string | null | undefined +): { path: string; query: Record } | null { + if (!route) return null; + let r = route.trim(); + if (!r || /^[a-z][a-z0-9+.-]*:/i.test(r) || r.startsWith('//')) return null; + r = r.replace(/^\/?dreamfactory\/dist\/?/, '').replace(/^#/, ''); + const q = r.indexOf('?'); + const rawPath = q >= 0 ? r.slice(0, q) : r; + const query: Record = {}; + if (q >= 0) { + new URLSearchParams(r.slice(q + 1)).forEach((v, k) => (query[k] = v)); + } + const path = `/${rawPath.replace(/^\/+/, '')}`; + return path === '/' ? null : { path, query }; +} + +/** yyyy-mm-dd in local time, for report file names. */ +function today(): string { + const d = new Date(); + const p = (n: number) => String(n).padStart(2, '0'); + return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`; +} + +/** How often a running scan is polled for progress. */ +export const SCAN_POLL_MS = 2000; + +@Injectable() +export class ComplianceStore implements OnDestroy { + private api = inject(DfComplianceApiService); + private masking = inject(DfDataMaskingApiService); + private snackbar = inject(DfSnackbarService); + private transloco = inject(TranslocoService); + + baselines: BaselineSummary[] | null = null; + readonly details = new Map(); + selectedId: string | null = null; + + loading = true; + loadFailed = false; + /** + * The server's reason when the list failed (e.g. 503 "Data Masking is + * not set up yet. Run: ..."); null for a generic failure. + */ + loadError: string | null = null; + detailFailed = false; + /** Write in flight: apply / stop tracking / export. */ + busy: 'apply' | 'stop' | ReportFormat | null = null; + /** Masking mode, for the "also turn masking on" choice; null = unknown. */ + maskingMode: MaskingMode | null = null; + /** The last masking overview read; null = not read (or failed). */ + private overview: MaskingOverview | null = null; + /** Apply started the first scan; shown until polling sees it finish. */ + private scanKicked = false; + private poll?: Subscription; + + /** Only the latest detail request may land. */ + private detailSeq = 0; + + /** + * No scan has found anything yet: none has finished, or only scans of + * empty databases (the server still refuses On, and Apply scans again). + */ + get neverScanned(): boolean { + const o = this.overview; + if (!o) return false; + const c = o.counts; + const found = + (o.scan?.columns ?? 0) > 0 || (!!c && c.masked + c.review + c.clear > 0); + return !o.scan?.finishedAt || !found; + } + + /** A scan is running (one Apply started, or one already under way). */ + get scanning(): boolean { + return this.scanKicked || this.overview?.scan?.status === 'running'; + } + + /** Scan progress 0-100, or null while the server can't say yet. */ + get scanProgress(): number | null { + const p = this.overview?.scan?.progress; + return this.overview?.scan?.status === 'running' && typeof p === 'number' + ? p + : null; + } + + /** + * Offer turning masking on with Apply: only once a scan has finished + * (the server refuses On before the first scan) and while it isn't on. + */ + get offerTurnOn(): boolean { + return ( + !!this.overview && + !this.neverScanned && + !this.scanKicked && + this.overview.mode !== 'on' + ); + } + + t(key: string, params?: Record): string { + return this.transloco.translate(`compliance.${key}`, params); + } + + get selected(): BaselineSummary | undefined { + return this.baselines?.find(b => b.id === this.selectedId); + } + + get detail(): BaselineDetail | undefined { + return this.selectedId ? this.details.get(this.selectedId) : undefined; + } + + /** Baselines grouped by vertical, verticals in catalog order. */ + get groups(): VerticalGroup[] { + const out = new Map(); + const sorted = [...(this.baselines ?? [])].sort( + (a, b) => verticalOrder(a.vertical) - verticalOrder(b.vertical) + ); + for (const b of sorted) { + let g = out.get(b.vertical); + if (!g) { + g = { + vertical: b.vertical, + icon: + VERTICALS.find(v => v.id === b.vertical)?.icon ?? 'verified_user', + baselines: [], + }; + out.set(b.vertical, g); + } + g.baselines.push(b); + } + return [...out.values()]; + } + + /** + * Load the list, then select `preferred` if it exists, else the first + * tracked baseline, else the first one. + */ + load(preferred?: string | null): void { + this.loading = true; + this.loadFailed = false; + this.loadError = null; + this.refreshMaskingMode(); + this.api.list().subscribe({ + next: list => { + this.baselines = list; + this.loading = false; + const pick = + list.find(b => b.id === preferred) ?? + list.find(b => b.adopted) ?? + list[0]; + if (pick) this.select(pick.id); + }, + error: e => { + this.loading = false; + this.loadFailed = true; + const message = normalizeError(e).message; + this.loadError = /^(errors|alerts)\./.test(message) ? null : message; + }, + }); + } + + select(id: string): void { + if (!this.baselines?.some(b => b.id === id)) return; + this.selectedId = id; + this.detailFailed = false; + this.loadDetail(id); + } + + /** + * (Re)load a baseline's checks; a cached copy stays up meanwhile. Every + * response is cached under its own id, so a slow one for a baseline the + * admin already left never shows in place of the selected one. + */ + loadDetail(id: string): void { + const n = ++this.detailSeq; + this.detailFailed = false; + this.api.get(id).subscribe({ + next: d => { + this.details.set(id, d); + this.patchSummary(d); + }, + error: () => { + if (n === this.detailSeq && !this.details.has(id)) { + this.detailFailed = true; + } + }, + }); + } + + /** Keep the rail's score and tracking dot in step with a fresh detail. */ + private patchSummary(d: BaselineDetail): void { + if (!this.baselines) return; + const counts = { pass: 0, fail: 0, manual: 0, total: d.controls.length }; + for (const c of d.controls) { + if (c.status === 'pass') counts.pass++; + else if (c.status === 'fail') counts.fail++; + else if (c.status === 'manual') counts.manual++; + } + this.baselines = this.baselines.map(b => + b.id === d.id ? { ...b, adopted: d.adopted, score: counts } : b + ); + } + + private refreshMaskingMode(): void { + this.masking.overview(true).subscribe({ + next: o => this.setOverview(o), + error: () => { + this.overview = null; + this.maskingMode = null; + }, + }); + } + + private setOverview(o: MaskingOverview): void { + this.overview = o; + this.maskingMode = o?.mode ?? null; + if (o?.scan?.status === 'running') this.watchScan(); + } + + /** + * Poll the scan until it finishes, then re-read the checks: the scan + * decides masking.scanned and every count-based check. + */ + private watchScan(): void { + if (this.poll) return; + this.poll = timer(SCAN_POLL_MS, SCAN_POLL_MS) + .pipe( + switchMap(() => + this.masking.overview(true).pipe(catchError(() => of(null))) + ) + ) + .subscribe(o => { + if (!o) return; // transient: keep polling + this.overview = o; + this.maskingMode = o.mode ?? null; + if (o.scan?.status !== 'running') { + this.stopPolling(); + this.scanKicked = false; + this.afterScan(); + } + }); + } + + private stopPolling(): void { + this.poll?.unsubscribe(); + this.poll = undefined; + } + + private afterScan(): void { + const keep = this.selectedId; + for (const id of [...this.details.keys()]) { + if (id !== keep) this.details.delete(id); + } + this.reloadList(); + if (keep) this.loadDetail(keep); + } + + ngOnDestroy(): void { + this.stopPolling(); + } + + /** + * Apply (and adopt) a baseline. Settings are instance-wide, so every + * other baseline's checks are stale afterwards. + */ + apply(id: string, turnOn: boolean): Observable { + this.busy = 'apply'; + return this.api.apply(id, turnOn).pipe( + tap({ + next: d => { + const name = d.name || this.nameOf(id); + this.details.clear(); + this.details.set(id, d); + this.reloadList(); + if (d.scanStarted) { + // Apply on a never-scanned instance starts the first scan. + this.scanKicked = true; + this.watchScan(); + } + this.refreshMaskingMode(); + this.toast( + d.scanStarted + ? 'toast.appliedScanning' + : turnOn + ? 'toast.appliedOn' + : 'toast.applied', + { name } + ); + }, + error: e => this.fail(e), + }), + finalize(() => (this.busy = null)) + ); + } + + /** What Apply would change with masking turned on too (read-only). */ + changesWithModeOn(id: string): Observable { + return this.api.changesWithModeOn(id); + } + + stopTracking(id: string): void { + if (this.busy) return; + this.busy = 'stop'; + const name = this.nameOf(id); + this.api + .stopTracking(id) + .pipe(finalize(() => (this.busy = null))) + .subscribe({ + next: () => { + const d = this.details.get(id); + if (d) this.details.set(id, { ...d, adopted: false }); + if (this.baselines) { + this.baselines = this.baselines.map(b => + b.id === id ? { ...b, adopted: false } : b + ); + } + this.loadDetail(id); + this.toast('toast.stopped', { name }); + }, + error: e => this.fail(e), + }); + } + + /** Download the evidence report exactly as the server rendered it. */ + exportReport(id: string, format: ReportFormat): void { + if (this.busy) return; + this.busy = format; + this.api + .report(id, format) + .pipe(finalize(() => (this.busy = null))) + .subscribe({ + next: text => + saveRawAsFile( + text, + `dreamfactory-${id}-baseline-${today()}.${format}`, + format + ), + error: e => this.fail(parseTextError(e)), + }); + } + + private reloadList(): void { + this.api.list().subscribe({ + next: list => (this.baselines = list), + error: () => undefined, + }); + } + + private nameOf(id: string): string { + return this.baselines?.find(b => b.id === id)?.name ?? id; + } + + toast(key: string, params?: Record): void { + this.snackbar.openSnackBar(this.t(key, params), 'success'); + } + + fail(e: unknown): void { + this.snackbar.openError(isAppError(e) ? e : normalizeError(e)); + } +} + +/** + * A text-typed request's error body is the raw JSON string. Parse it back + * into DreamFactory's error envelope so the toast carries the server's + * message, not a generic status line. Works on the raw HttpErrorResponse + * and on the AppError the error interceptor already made of it (toast-off + * requests reach the store normalized, with the string body in `raw`). + */ +export function parseTextError(e: unknown): unknown { + if (isAppError(e)) { + const body = typeof e.raw === 'string' ? parseJson(e.raw) : null; + if (!body) return e; + const parsed = normalizeError( + new HttpErrorResponse({ error: body, status: e.status, url: e.url }), + { url: e.url, method: e.method } + ); + return parsed.message.startsWith('errors.') ? e : parsed; + } + if (!(e instanceof HttpErrorResponse) || typeof e.error !== 'string') { + return e; + } + const body = parseJson(e.error); + return body + ? new HttpErrorResponse({ + error: body, + headers: e.headers, + status: e.status, + statusText: e.statusText, + url: e.url ?? undefined, + }) + : e; +} + +function parseJson(text: string): object | null { + try { + const body: unknown = JSON.parse(text); + return body && typeof body === 'object' ? body : null; + } catch { + return null; + } +} diff --git a/src/app/adf-compliance/compliance.types.ts b/src/app/adf-compliance/compliance.types.ts new file mode 100644 index 000000000..164c1cceb --- /dev/null +++ b/src/app/adf-compliance/compliance.types.ts @@ -0,0 +1,72 @@ +/** + * Compliance baseline API types, as the UI sees them: responses pass through + * the case interceptor, so snake_case wire keys arrive camelCased + * (adopted_at -> adoptedAt). Backend contract, section 8. + */ + +export type ControlStatus = 'pass' | 'fail' | 'manual' | 'na'; + +export interface BaselineScore { + pass: number; + fail: number; + manual: number; + total: number; +} + +/** One row of GET /system/security_baseline. */ +export interface BaselineSummary { + id: string; + name: string; + /** healthcare | payments | banking | government | privacy | general */ + vertical: string; + frameworks: string[]; + adopted: boolean; + score: BaselineScore; +} + +export interface ControlFix { + /** + * apply: Apply fixes it. link: the admin fixes it at `route`. + * none: nothing DreamFactory can do (or nothing to do). + */ + kind: 'apply' | 'link' | 'none'; + route: string | null; +} + +export interface BaselineControl { + id: string; + title: string; + /** Framework control references, e.g. "45 CFR 164.312(b)". */ + refs: string[]; + status: ControlStatus; + /** Why it failed or what was found; never data values. */ + detail: string | null; + fix: ControlFix; +} + +/** One line of what Apply would change right now. */ +export interface BaselineChange { + id: string; + text: string; +} + +/** GET /system/security_baseline/{id} (and the apply response). */ +export interface BaselineDetail { + id: string; + name: string; + vertical: string; + frameworks: string[]; + adopted: boolean; + adoptedAt: string | null; + appliedAt: string | null; + controls: BaselineControl[]; + /** Exactly what Apply would change now; empty when nothing to do. */ + changes: BaselineChange[]; + /** + * Apply response only (`scan_started`): Apply ran on an instance that + * was never scanned, so it started the first scan. + */ + scanStarted?: boolean; +} + +export type ReportFormat = 'json' | 'csv'; diff --git a/src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.spec.ts b/src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.spec.ts new file mode 100644 index 000000000..69b7320ed --- /dev/null +++ b/src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.spec.ts @@ -0,0 +1,151 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Subject, of, throwError } from 'rxjs'; +import { + ApplyDialogData, + DfComplianceApplyDialogComponent, +} from './df-compliance-apply-dialog.component'; +import { cpTransloco } from '../compliance-testing'; +import { BaselineChange } from '../compliance.types'; + +const RULES: BaselineChange[] = [ + { id: 'rule:0', text: 'Add a detection rule: medical record numbers.' }, +]; +const WITH_ON: BaselineChange[] = [ + { id: 'mode', text: 'Turn masking on (now Preview).' }, + ...RULES, +]; + +describe('DfComplianceApplyDialogComponent', () => { + let fixture: ComponentFixture; + + const el = () => fixture.nativeElement as HTMLElement; + const listed = () => + Array.from( + el().querySelectorAll('[data-testid="cp-apply-changes"] li') + ).map(li => li.textContent?.trim()); + const toggleTurnOn = () => { + ( + el().querySelector( + '[data-testid="cp-apply-turn-on"] input' + ) as HTMLInputElement + ).click(); + fixture.detectChanges(); + }; + + function create(data: ApplyDialogData) { + TestBed.configureTestingModule({ + imports: [ + DfComplianceApplyDialogComponent, + NoopAnimationsModule, + cpTransloco(), + ], + providers: [ + { provide: MAT_DIALOG_DATA, useValue: data }, + { provide: MatDialogRef, useValue: { close: jest.fn() } }, + ], + }); + fixture = TestBed.createComponent(DfComplianceApplyDialogComponent); + fixture.detectChanges(); + } + + it('lists exactly the server changes, and the mode-on list while turning on', () => { + create({ + name: 'HIPAA', + changes: RULES, + offerTurnOn: true, + changesOn$: of(WITH_ON), + }); + expect(el().textContent).toContain('Apply HIPAA baseline?'); + expect( + el().querySelector('[data-testid="cp-apply-turn-on"]')?.textContent + ).toContain('Mask values now (masking On)'); + expect(listed()).toEqual([RULES[0].text]); + toggleTurnOn(); + expect(listed()).toEqual(WITH_ON.map(c => c.text)); + toggleTurnOn(); + expect(listed()).toEqual([RULES[0].text]); + }); + + it('shows the turn-on change even when nothing else would change', () => { + create({ + name: 'GDPR', + changes: [], + offerTurnOn: true, + changesOn$: of([WITH_ON[0]]), + }); + expect(el().querySelector('[data-testid="cp-apply-none"]')).toBeTruthy(); + toggleTurnOn(); + expect(listed()).toEqual([WITH_ON[0].text]); + }); + + it('keeps the plain list until the mode-on list arrives, or if it fails', () => { + const later = new Subject(); + create({ + name: 'HIPAA', + changes: RULES, + offerTurnOn: true, + changesOn$: later, + }); + toggleTurnOn(); + expect(listed()).toEqual([RULES[0].text]); + later.next(WITH_ON); + fixture.detectChanges(); + expect(listed()).toEqual(WITH_ON.map(c => c.text)); + + TestBed.resetTestingModule(); + create({ + name: 'HIPAA', + changes: RULES, + offerTurnOn: true, + changesOn$: throwError(() => new Error('offline')), + }); + toggleTurnOn(); + expect(listed()).toEqual([RULES[0].text]); + }); + + it('starts with turning masking on ticked when asked (the Fix of "Masking is on")', async () => { + create({ + name: 'HIPAA', + changes: [], + offerTurnOn: true, + turnOn: true, + changesOn$: of([WITH_ON[0]]), + }); + // ngModel writes the initial value asynchronously. + await fixture.whenStable(); + fixture.detectChanges(); + const box = el().querySelector( + '[data-testid="cp-apply-turn-on"] input' + ) as HTMLInputElement; + expect(box.checked).toBe(true); + expect(listed()).toEqual([WITH_ON[0].text]); + expect(fixture.componentInstance.turnOn).toBe(true); + }); + + it('never says "settings already match" while turning masking on', () => { + // The mode-on list is still loading (or failed): the plain list is empty. + create({ + name: 'GDPR', + changes: [], + offerTurnOn: true, + changesOn$: new Subject(), + }); + expect(el().querySelector('[data-testid="cp-apply-none"]')).toBeTruthy(); + toggleTurnOn(); + expect(el().querySelector('[data-testid="cp-apply-none"]')).toBeNull(); + }); + + it('never presets turning masking on when it is not offered', () => { + create({ name: 'HIPAA', changes: RULES, offerTurnOn: false, turnOn: true }); + expect(fixture.componentInstance.turnOn).toBe(false); + expect(el().querySelector('[data-testid="cp-apply-turn-on"]')).toBeNull(); + }); + + it('offers no turn-on choice when masking is already on', () => { + create({ name: 'HIPAA', changes: RULES, offerTurnOn: false }); + expect(el().querySelector('[data-testid="cp-apply-turn-on"]')).toBeNull(); + expect(listed()).toEqual([RULES[0].text]); + }); +}); diff --git a/src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.ts b/src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.ts new file mode 100644 index 000000000..acb5b3bed --- /dev/null +++ b/src/app/adf-compliance/df-compliance/df-compliance-apply-dialog.component.ts @@ -0,0 +1,175 @@ +import { Component, Inject, OnDestroy } from '@angular/core'; +import { NgFor, NgIf } from '@angular/common'; +import { FormsModule } from '@angular/forms'; +import { MatButtonModule } from '@angular/material/button'; +import { MatCheckboxModule } from '@angular/material/checkbox'; +import { MAT_DIALOG_DATA, MatDialogModule } from '@angular/material/dialog'; +import { TranslocoPipe } from '@ngneat/transloco'; +import { Observable, Subscription } from 'rxjs'; +import { BaselineChange } from '../compliance.types'; + +export interface ApplyDialogData { + name: string; + /** Exactly the server's `changes`, in its order. */ + changes: BaselineChange[]; + /** + * Offer turning masking on: a scan has finished (the server refuses On + * before the first scan) and masking is not on already. + */ + offerTurnOn: boolean; + /** Start with "turn masking on" ticked (the Fix of "Masking is on"). */ + turnOn?: boolean; + /** + * The server's list for Apply with masking turned on (`changes` plus the + * mode change), shown while "also turn masking on" is checked. + */ + changesOn$?: Observable; +} + +export interface ApplyDialogResult { + turnOn: boolean; +} + +/** + * Apply confirm: the list of what will change is the message, exactly as + * the server reports it for the choice made (with or without turning + * masking on). Closes with `{ turnOn }` on Apply, undefined on Cancel. + */ +@Component({ + selector: 'df-compliance-apply-dialog', + standalone: true, + imports: [ + NgIf, + NgFor, + FormsModule, + MatButtonModule, + MatCheckboxModule, + MatDialogModule, + TranslocoPipe, + ], + template: ` +

+ {{ 'compliance.applyDialog.title' | transloco: { name: data.name } }} +

+
+
    +
  • + {{ c.text }} +
    • +
+ + +

+ {{ 'compliance.applyDialog.none' | transloco }} +

+ + + {{ 'compliance.applyDialog.turnOn' | transloco }} + +
+
+ + +
+ `, + styles: [ + ` + @use '../../shared/components/df-dialog-surface' as *; + @include df-dialog-surface; + + .body { + color: var(--df-text-2); + font-size: var(--df-font-size-sm); + max-width: 480px; + } + + .changes { + display: flex; + flex-direction: column; + gap: var(--df-space-2); + list-style: none; + margin: 0 0 var(--df-space-4); + max-height: 40vh; + overflow-y: auto; + padding: 0; + } + + .changes li { + color: var(--df-text); + line-height: var(--df-lh-base); + padding-left: var(--df-space-4); + position: relative; + } + + // A small accent tick per change: reads as a list of edits. + .changes li::before { + background: var(--df-accent); + border-radius: 50%; + content: ''; + height: 6px; + left: 2px; + position: absolute; + top: 0.6em; + width: 6px; + } + + .none { + margin: 0 0 var(--df-space-4); + } + + .turn-on { + margin-left: calc(-1 * var(--df-space-2)); + } + `, + ], +}) +export class DfComplianceApplyDialogComponent implements OnDestroy { + turnOn: boolean; + /** The server's list with masking turned on; null until it arrives. */ + private changesOn: BaselineChange[] | null = null; + private sub?: Subscription; + + constructor(@Inject(MAT_DIALOG_DATA) public data: ApplyDialogData) { + this.turnOn = !!(data.offerTurnOn && data.turnOn); + if (data.offerTurnOn && data.changesOn$) { + // A failed read keeps the plain list: Apply still does what it says. + this.sub = data.changesOn$.subscribe({ + next: c => (this.changesOn = c), + error: () => undefined, + }); + } + } + + /** What Apply will change for the current choice. */ + get shown(): BaselineChange[] { + return this.turnOn && this.changesOn ? this.changesOn : this.data.changes; + } + + ngOnDestroy(): void { + this.sub?.unsubscribe(); + } + + trackChange = (_: number, c: BaselineChange) => c.id; +} diff --git a/src/app/adf-compliance/df-compliance/df-compliance.component.html b/src/app/adf-compliance/df-compliance/df-compliance.component.html new file mode 100644 index 000000000..2371d404f --- /dev/null +++ b/src/app/adf-compliance/df-compliance/df-compliance.component.html @@ -0,0 +1,313 @@ + + + +
+ + + + + + + + + + + + + + {{ 'compliance.baseline' | transloco }} + + {{ store.selected?.name }} + + + {{ b.name }} + {{ railScore(b) }} + + + + + +
+ + +
+ +
+
+

{{ title(d) }}

+ + + {{ 'compliance.tracking' | transloco }} + +
+
+ + + + + + + +
+
+ +

{{ mapped(d) }}

+ +
+ + {{ 'compliance.scanning' | transloco }} + {{ store.scanProgress }}% + + +
+ +
+ + {{ s.pass }} + {{ 'compliance.of' | transloco: { total: s.checked } }} + + +
+ +
    + + + +
+ +
+ + +
    + + + +
+ +
+ + + + + + + + +
+
+ +
+ + + + +
  • + + + {{ statusLabel(c) }}: + + + {{ c.title }} + {{ c.detail }} + + + {{ + r + }} + + + + + {{ 'compliance.fix' | transloco }} + + + + + + +
    • +     diff --git a/src/app/adf-compliance/df-compliance/df-compliance.component.scss b/src/app/adf-compliance/df-compliance/df-compliance.component.scss new file mode 100644 index 000000000..38a3d1739 --- /dev/null +++ b/src/app/adf-compliance/df-compliance/df-compliance.component.scss @@ -0,0 +1,527 @@ +// Compliance page: a rail of baselines grouped by vertical (the same +// grammar as Data Masking's Columns rail) and the selected baseline's +// score and checks. Tokens only: light, dark and phosphor resolve from +// src/styles.scss. + +:host { + display: block; +} + +.cp { + margin: 0 auto; + max-width: var(--df-content-max); +} + +.sr-only { + clip: rect(0 0 0 0); + height: 1px; + overflow: hidden; + position: absolute; + white-space: nowrap; + width: 1px; +} + +// ------------------------------------------------------------------ panes + +.baseline-select { + display: none; + margin-bottom: var(--df-space-3); + width: 100%; +} + +.opt-score { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + margin-left: var(--df-space-2); +} + +.panes { + align-items: start; + display: grid; + gap: var(--df-space-5); + grid-template-columns: 248px minmax(0, 1fr); +} + +// ------------------------------------------------------------------- rail + +.rail { + border-right: 1px solid var(--df-border-2); + padding-right: var(--df-space-3); + position: sticky; + top: 0; +} + +.rail-group + .rail-group { + margin-top: var(--df-space-4); +} + +.rail-vertical { + align-items: center; + color: var(--df-text-muted); + display: flex; + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + gap: 6px; + margin: 0 0 var(--df-space-1); + padding: 0 var(--df-space-2); + + mat-icon { + font-size: 14px; + height: 14px; + width: 14px; + } +} + +.rail-row { + align-items: center; + background: transparent; + border: 0; + border-radius: var(--df-radius-sm); + color: var(--df-text-2); + cursor: pointer; + display: grid; + font: inherit; + font-size: var(--df-font-size-sm); + gap: var(--df-space-2); + grid-template-columns: minmax(0, 1fr) auto auto; + height: 32px; + padding: 0 var(--df-space-2); + text-align: left; + width: 100%; + + &:hover { + background: var(--df-hover); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + &.is-selected { + background: var(--df-accent-soft); + color: var(--df-text); + font-weight: var(--df-font-weight-medium); + } +} + +.rail-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +// Tracked baselines carry a small accent dot; spoken via the row label. +.rail-tracking { + background: var(--df-accent); + border-radius: 50%; + height: 6px; + width: 6px; +} + +.rail-score { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + min-width: 34px; + text-align: right; + + .is-selected & { + color: var(--df-text-2); + } +} + +// ----------------------------------------------------------------- detail + +.detail { + min-width: 0; + container-type: inline-size; +} + +.head { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: var(--df-space-3); + justify-content: space-between; + margin-bottom: var(--df-space-2); + min-height: 40px; +} + +.title-row { + align-items: center; + display: flex; + gap: var(--df-space-3); + min-width: 0; +} + +.title { + color: var(--df-text); + font-size: var(--df-font-size-xl); + font-weight: var(--df-font-weight-heading); + letter-spacing: var(--df-tracking-tight); + margin: 0; +} + +.tracking { + align-items: center; + background: var(--df-accent-soft); + border-radius: 999px; + color: var(--df-accent-strong); + cursor: default; + display: inline-flex; + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + gap: 6px; + height: 22px; + padding: 0 var(--df-space-2); + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } +} + +.tracking-dot { + background: var(--df-accent); + border-radius: 50%; + height: 6px; + width: 6px; +} + +.actions { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: var(--df-space-2); + + mat-icon { + font-size: 18px; + height: 18px; + width: 18px; + } +} + +.stop { + color: var(--df-text-2); +} + +// Pending change count riding the Apply button. +.apply-n { + background: var(--df-accent-contrast); + border-radius: 999px; + color: var(--df-accent-strong); + display: inline-block; + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-heading); + line-height: 18px; + margin-left: 6px; + min-width: 18px; + padding: 0 5px; + text-align: center; +} + +// ------------------------------------------------------------------ score + +// What the baseline maps to, and that it is not a certification. +.mapped { + color: var(--df-text-muted); + font-size: var(--df-font-size-sm); + line-height: var(--df-lh-base); + margin: calc(-1 * var(--df-space-1)) 0 var(--df-space-3); +} + +.scan { + color: var(--df-text-2); + display: flex; + flex-direction: column; + font-size: var(--df-font-size-sm); + gap: var(--df-space-1); + margin-bottom: var(--df-space-3); + max-width: 360px; +} + +.score { + align-items: center; + display: flex; + gap: var(--df-space-3); + margin-bottom: var(--df-space-4); +} + +.score-num { + color: var(--df-text-2); + font-size: var(--df-font-size-sm); + white-space: nowrap; + + strong { + color: var(--df-text); + font-size: var(--df-font-size-xl); + font-weight: var(--df-font-weight-heading); + } +} + +.meter { + background: var(--df-surface-2); + border: 1px solid var(--df-border-2); + border-radius: 999px; + display: block; + flex: 0 1 240px; + height: 6px; + overflow: hidden; + + span { + background: var(--df-success); + border-radius: inherit; + display: block; + height: 100%; + transition: width var(--df-duration-standard) var(--df-ease-standard); + } +} + +// --------------------------------------------------------------- controls + +.controls { + border-top: 1px solid var(--df-border-2); + list-style: none; + margin: 0; + padding: 0; +} + +.control { + align-items: center; + border-bottom: 1px solid var(--df-border-2); + column-gap: var(--df-space-3); + display: grid; + grid-template-columns: 20px minmax(0, 1fr) auto 48px; + min-height: var(--df-row-height); + padding: 6px var(--df-space-2); +} + +.mark { + display: inline-flex; + + mat-icon { + font-size: 18px; + height: 18px; + width: 18px; + } + + .is-pass & { + color: var(--df-success); + } + + .is-fail & { + color: var(--df-danger); + } + + .is-manual & { + color: var(--df-text-muted); + } + + .is-na & { + color: var(--df-text-faint); + } +} + +.body { + display: flex; + flex-direction: column; + min-width: 0; +} + +.c-title { + color: var(--df-text); + font-size: var(--df-font-size-sm); + line-height: var(--df-lh-base); + + .is-na & { + color: var(--df-text-muted); + } +} + +.c-detail { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + line-height: var(--df-lh-base); +} + +.refs { + display: flex; + flex-wrap: wrap; + gap: var(--df-space-1); + justify-content: flex-end; + max-width: 340px; +} + +.ref { + background: var(--df-surface-2); + border: 1px solid var(--df-border-2); + border-radius: var(--df-radius-sm); + color: var(--df-text-2); + font-family: var(--df-font-mono); + font-size: 11px; + line-height: 18px; + padding: 0 6px; + white-space: nowrap; +} + +.act { + display: flex; + justify-content: flex-end; +} + +.fix { + border-radius: var(--df-radius-sm); + color: var(--df-accent-strong); + font-size: var(--df-font-size-sm); + font-weight: var(--df-font-weight-medium); + padding: 2px 6px; + text-decoration: none; + + &:hover { + background: var(--df-accent-soft); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } +} + +// Apply fixes: the same quiet action as the links, as a button. +button.fix { + background: none; + border: 0; + cursor: pointer; + font-family: inherit; + line-height: inherit; + + &:disabled { + color: var(--df-text-muted); + cursor: default; + } +} + +// ----------------------------------------------------------------- manual + +.manual { + margin-top: var(--df-space-4); +} + +.manual-toggle { + align-items: center; + background: transparent; + border: 0; + border-radius: var(--df-radius-sm); + color: var(--df-text-2); + cursor: pointer; + display: inline-flex; + font: inherit; + font-size: var(--df-font-size-sm); + gap: var(--df-space-1); + height: 30px; + padding: 0 var(--df-space-2) 0 var(--df-space-1); + + &:hover { + background: var(--df-hover); + color: var(--df-text); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + mat-icon { + font-size: 18px; + height: 18px; + transition: transform var(--df-duration-standard) var(--df-ease-standard); + width: 18px; + } + + &[aria-expanded='true'] mat-icon { + transform: rotate(90deg); + } +} + +.manual-n { + background: var(--df-surface-2); + border-radius: 999px; + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + line-height: 18px; + min-width: 18px; + padding: 0 5px; + text-align: center; +} + +.controls.is-manual { + margin-top: var(--df-space-2); +} + +// ---------------------------------------------------------------- narrow + +@media (max-width: 1240px) { + .panes { + gap: var(--df-space-4); + grid-template-columns: 216px minmax(0, 1fr); + } +} + +@media (max-width: 760px) { + .baseline-select { + display: block; + } + + .panes { + display: block; + } + + .rail { + display: none; + } +} + +// Checks reflow on the detail's own width: refs drop under the title. +@container (max-width: 640px) { + .control { + grid-template-areas: + 'mark body act' + '. refs refs'; + grid-template-columns: 20px minmax(0, 1fr) auto; + row-gap: var(--df-space-1); + } + + .mark { + align-self: start; + grid-area: mark; + padding-top: 1px; + } + + .body { + grid-area: body; + } + + .act { + align-self: start; + grid-area: act; + } + + .refs { + grid-area: refs; + justify-content: flex-start; + max-width: none; + + &:empty { + display: none; + } + } + + .head { + align-items: flex-start; + } + + // The actions wrap under the title here: no pull-up. + .mapped { + margin-top: 0; + } +} diff --git a/src/app/adf-compliance/df-compliance/df-compliance.component.spec.ts b/src/app/adf-compliance/df-compliance/df-compliance.component.spec.ts new file mode 100644 index 000000000..1bed9d4be --- /dev/null +++ b/src/app/adf-compliance/df-compliance/df-compliance.component.spec.ts @@ -0,0 +1,398 @@ +import { Component, Input } from '@angular/core'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { DfPaywallComponent } from 'src/app/shared/components/df-paywall/df-paywall.component'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { By } from '@angular/platform-browser'; +import { + ActivatedRoute, + ParamMap, + Router, + convertToParamMap, + provideRouter, +} from '@angular/router'; +import { MatDialog } from '@angular/material/dialog'; +import { BehaviorSubject, of, throwError } from 'rxjs'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { DfDataMaskingApiService } from 'src/app/adf-data-masking/data-masking-api.service'; +import * as fileUtils from 'src/app/shared/utilities/file'; +import { DfComplianceComponent } from './df-compliance.component'; +import { DfComplianceApiService } from '../compliance-api.service'; +import { + control, + cpTransloco, + detail, + overview, + summary, +} from '../compliance-testing'; +import { BaselineDetail } from '../compliance.types'; + +jest.mock('src/app/shared/utilities/file', () => ({ + saveRawAsFile: jest.fn(), +})); + +/** The shared paywall tracks a hit and boots Calendly: stubbed here. */ +@Component({ selector: 'df-paywall', standalone: true, template: '' }) +class StubPaywallComponent { + @Input() serviceName = ''; +} + +describe('DfComplianceComponent', () => { + let fixture: ComponentFixture; + let api: Record; + let masking: { overview: jest.Mock }; + let dialog: { open: jest.Mock }; + let closeWith: unknown; + let query: BehaviorSubject; + let hipaa: BaselineDetail; + + const el = () => fixture.nativeElement as HTMLElement; + const q = (id: string) => + el().querySelector(`[data-testid="${id}"]`) as HTMLElement | null; + + function create(params: Record = {}, access = 'ok') { + query = new BehaviorSubject(convertToParamMap(params)); + TestBed.overrideProvider(ActivatedRoute, { + useValue: { + data: of({ data: access }), + queryParamMap: query, + snapshot: { queryParamMap: convertToParamMap(params) }, + }, + }); + fixture = TestBed.createComponent(DfComplianceComponent); + fixture.detectChanges(); + } + + beforeEach(() => { + hipaa = detail(); + api = { + list: jest.fn(() => + of([ + summary(), + summary({ + id: 'pci_dss', + name: 'PCI DSS', + vertical: 'payments', + adopted: true, + score: { pass: 7, fail: 5, manual: 1, total: 13 }, + }), + summary({ id: 'soc2', name: 'SOC 2', vertical: 'general' }), + ]) + ), + get: jest.fn((id: string) => + of( + id === 'hipaa' + ? hipaa + : detail({ id, name: id === 'soc2' ? 'SOC 2' : 'PCI DSS' }) + ) + ), + apply: jest.fn(() => of(detail({ adopted: true, changes: [] }))), + changesWithModeOn: jest.fn(() => + of([{ id: 'mode', text: 'Turn masking on (now Preview).' }]) + ), + stopTracking: jest.fn(() => of({})), + report: jest.fn(() => of('control,status\n')), + }; + masking = { overview: jest.fn(() => of(overview('preview'))) }; + closeWith = undefined; + dialog = { + open: jest.fn(() => ({ afterClosed: () => of(closeWith) })), + }; + TestBed.configureTestingModule({ + imports: [DfComplianceComponent, NoopAnimationsModule, cpTransloco()], + providers: [ + provideRouter([]), + { provide: DfComplianceApiService, useValue: api }, + { provide: DfDataMaskingApiService, useValue: masking }, + { + provide: DfSnackbarService, + useValue: { openSnackBar: jest.fn(), openError: jest.fn() }, + }, + ], + }); + TestBed.overrideProvider(MatDialog, { useValue: dialog }); + }); + + it('groups baselines by vertical in the rail, with scores', () => { + create({ baseline: 'hipaa' }); + const heads = Array.from(el().querySelectorAll('.rail-vertical')).map(h => + h.textContent?.replace(/\s+/g, ' ').trim() + ); + expect(heads).toEqual([ + 'health_and_safety Healthcare', + 'credit_card Payments', + 'verified_user General', + ]); + const pci = q('cp-rail-pci_dss')!; + expect(pci.querySelector('.rail-score')?.textContent).toBe('7/12'); + expect(pci.querySelector('.rail-tracking')).toBeTruthy(); + expect(pci.getAttribute('aria-label')).toBe( + 'PCI DSS, Tracking, 7 of 12 checks pass' + ); + expect(q('cp-rail-hipaa')!.getAttribute('aria-current')).toBe('true'); + }); + + it('shows the score of automated checks only', () => { + create({ baseline: 'hipaa' }); + expect(q('cp-score')!.textContent?.replace(/\s+/g, ' ').trim()).toBe( + '1 of 3' + ); + const meter = el().querySelector('[role="meter"]')!; + expect(meter.getAttribute('aria-valuenow')).toBe('1'); + expect(meter.getAttribute('aria-valuemax')).toBe('3'); + expect(meter.getAttribute('aria-label')).toBe('1 of 3 checks pass'); + }); + + it('gives each check a spoken status, refs as chips and at most one action', () => { + create({ baseline: 'hipaa' }); + const review = q('cp-control-masking.review')!; + expect(review.querySelector('.sr-only')?.textContent).toBe('Fails:'); + expect(review.querySelector('.c-detail')?.textContent).toBe( + '24 columns to review' + ); + expect( + Array.from(review.querySelectorAll('.ref')).map(r => r.textContent) + ).toEqual(['45 CFR 164.514(b)']); + const fix = review.querySelector('[data-testid="cp-fix"]')!; + expect(fix.getAttribute('href')).toBe( + '/api-security/data-masking?tab=columns' + ); + expect(fix.getAttribute('aria-label')).toBe('Fix: Review queue is empty'); + // Apply fixes it: a Fix button that opens Apply. Passing: no action. + const applyFix = q('cp-control-access.ai')!.querySelector( + '[data-testid="cp-fix"]' + )!; + expect(applyFix.tagName).toBe('BUTTON'); + (applyFix as HTMLButtonElement).click(); + expect(dialog.open.mock.calls[0][1].data.turnOn).toBe(false); + expect( + q('cp-control-masking.mode')!.querySelector('[data-testid="cp-fix"]') + ).toBeNull(); + // n/a trails the checks that count. + const ids = Array.from( + q('cp-controls')!.querySelectorAll('[data-testid^="cp-control-"]') + ).map(li => li.getAttribute('data-testid')); + expect(ids[ids.length - 1]).toBe('cp-control-platform.rate_limit'); + }); + + it('keeps manual checks collapsed at the end until asked', () => { + create({ baseline: 'hipaa' }); + expect(q('cp-control-manual.encryption')).toBeNull(); + const toggle = q('cp-manual-toggle')!; + expect(toggle.getAttribute('aria-expanded')).toBe('false'); + expect(toggle.textContent).toContain('Manual checks'); + toggle.click(); + fixture.detectChanges(); + expect(toggle.getAttribute('aria-expanded')).toBe('true'); + expect(q('cp-control-manual.encryption')).toBeTruthy(); + }); + + it('Apply confirms with exactly the server changes and the turn-on option', () => { + create({ baseline: 'hipaa' }); + expect(q('cp-apply')!.textContent).toContain('2'); + closeWith = { turnOn: true }; + q('cp-apply')!.click(); + const data = dialog.open.mock.calls[0][1].data; + expect(data.name).toBe('HIPAA'); + expect(data.changes).toEqual(hipaa.changes); + expect(data.offerTurnOn).toBe(true); + expect(api['apply']).toHaveBeenCalledWith('hipaa', true); + }); + + it('hands the dialog what Apply would change with masking turned on', () => { + create({ baseline: 'hipaa' }); + q('cp-apply')!.click(); + const data = dialog.open.mock.calls[0][1].data; + let shown: unknown; + data.changesOn$.subscribe((c: unknown) => (shown = c)); + expect(api['changesWithModeOn']).toHaveBeenCalledWith('hipaa'); + expect(shown).toEqual([ + { id: 'mode', text: 'Turn masking on (now Preview).' }, + ]); + }); + + it('does not offer turning masking on when it already is', () => { + masking.overview.mockReturnValue(of(overview('on'))); + create({ baseline: 'hipaa' }); + q('cp-apply')!.click(); + expect(dialog.open.mock.calls[0][1].data.offerTurnOn).toBe(false); + expect(dialog.open.mock.calls[0][1].data.changesOn$).toBeUndefined(); + // Cancel applies nothing. + expect(api['apply']).not.toHaveBeenCalled(); + }); + + it('a tracked baseline with nothing to change or fix has no Apply, but Stop tracking', () => { + hipaa = detail({ + adopted: true, + adoptedAt: '2026-10-20T10:00:00Z', + changes: [], + controls: detail().controls.map(c => + c.fix.kind === 'apply' ? { ...c, status: 'pass' as const } : c + ), + }); + create({ baseline: 'hipaa' }); + expect(q('cp-apply')).toBeNull(); + expect(q('cp-tracking')!.textContent).toContain('Tracking'); + q('cp-stop')!.click(); + expect(api['stopTracking']).toHaveBeenCalledWith('hipaa'); + }); + + it('exports from the menu', () => { + create({ baseline: 'hipaa' }); + q('cp-export')!.click(); + fixture.detectChanges(); + const csv = document.querySelector( + '[data-testid="cp-export-csv"]' + ) as HTMLButtonElement; + expect(csv).toBeTruthy(); + csv.click(); + expect(api['report']).toHaveBeenCalledWith('hipaa', 'csv'); + expect(fileUtils.saveRawAsFile).toHaveBeenCalled(); + }); + + it('follows ?baseline= and writes the selection back to it', () => { + create({ baseline: 'hipaa' }); + const router = TestBed.inject(Router); + const nav = jest.spyOn(router, 'navigate').mockResolvedValue(true); + q('cp-rail-soc2')!.click(); + fixture.detectChanges(); + expect(nav).toHaveBeenCalledWith( + [], + expect.objectContaining({ + queryParams: { baseline: 'soc2' }, + replaceUrl: true, + }) + ); + expect(el().querySelector('#cp-title')?.textContent).toBe('SOC 2 baseline'); + query.next(convertToParamMap({ baseline: 'pci_dss' })); + fixture.detectChanges(); + expect(el().querySelector('#cp-title')?.textContent).toBe( + 'PCI DSS baseline' + ); + }); + + it('titles the page as a baseline, names what it maps to and that it is not a certification', () => { + hipaa = detail({ + frameworks: ['HIPAA Security Rule', 'HIPAA Privacy Rule'], + }); + create({ baseline: 'hipaa' }); + expect(el().querySelector('#cp-title')?.textContent).toBe('HIPAA baseline'); + expect(q('cp-mapped')!.textContent?.trim()).toBe( + 'Mapped to HIPAA Security Rule, HIPAA Privacy Rule · Not a certification' + ); + }); + + describe('a failing "Masking is on" check (fix kind apply)', () => { + // What the real backend returns after a default Apply on an instance in + // Preview: adopted, nothing left in `changes`, masking.mode failing with + // fix {kind:'apply', route:null} (GET /system/security_baseline/soc2). + beforeEach(() => { + hipaa = detail({ + adopted: true, + adoptedAt: '2026-10-29T06:00:00Z', + changes: [], + controls: [ + control({ + id: 'masking.mode', + title: 'Masking is on', + status: 'fail', + detail: 'Masking is in Preview: values are not masked yet.', + fix: { kind: 'apply', route: null }, + }), + control({ id: 'access.ai', title: 'AI masked', status: 'pass' }), + ], + }); + }); + + it('keeps Apply and gives the check a Fix that turns masking on', () => { + create({ baseline: 'hipaa' }); + expect(q('cp-apply')).toBeTruthy(); + const row = q('cp-control-masking.mode')!; + expect(row.classList).toContain('is-fail'); + const fix = row.querySelector('[data-testid="cp-fix"]') as HTMLElement; + expect(fix).toBeTruthy(); + closeWith = { turnOn: true }; + fix.click(); + const data = dialog.open.mock.calls[0][1].data; + expect(data.offerTurnOn).toBe(true); + expect(data.turnOn).toBe(true); + expect(api['apply']).toHaveBeenCalledWith('hipaa', true); + }); + + it('the header Apply leaves "turn masking on" for the admin to tick', () => { + create({ baseline: 'hipaa' }); + q('cp-apply')!.click(); + expect(dialog.open.mock.calls[0][1].data.turnOn).toBe(false); + }); + + it('before the first scan, never offers turning masking on; the Fix goes to Data Masking', () => { + masking.overview.mockReturnValue( + of(overview('preview', { finishedAt: null, columns: 0 })) + ); + create({ baseline: 'hipaa' }); + const fix = q('cp-control-masking.mode')!.querySelector( + '[data-testid="cp-fix"]' + )!; + expect(fix.tagName).toBe('A'); + expect(fix.getAttribute('href')).toBe('/api-security/data-masking'); + q('cp-apply')!.click(); + const data = dialog.open.mock.calls[0][1].data; + expect(data.offerTurnOn).toBe(false); + expect(data.turnOn).toBe(false); + expect(data.changesOn$).toBeUndefined(); + }); + }); + + it('shows the first scan Apply started, with its progress', () => { + masking.overview.mockReturnValue( + of(overview('preview', { finishedAt: null, columns: 0 })) + ); + create({ baseline: 'hipaa' }); + expect(q('cp-scanning')).toBeNull(); + api['apply'].mockReturnValue( + of({ ...detail({ adopted: true, changes: [] }), scanStarted: true }) + ); + masking.overview.mockReturnValue( + of( + overview('preview', { + status: 'running', + progress: 30, + finishedAt: null, + }) + ) + ); + closeWith = { turnOn: false }; + q('cp-apply')!.click(); + fixture.detectChanges(); + expect(q('cp-scanning')!.textContent).toContain('Scanning databases…'); + expect(q('cp-scanning')!.textContent).toContain('30%'); + const snack = TestBed.inject(DfSnackbarService) as unknown as { + openSnackBar: jest.Mock; + }; + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'HIPAA baseline applied. Scanning…', + 'success' + ); + fixture.destroy(); + }); + + it('without the security_baseline resource shows the shared paywall, on any license', () => { + TestBed.overrideComponent(DfComplianceComponent, { + remove: { imports: [DfPaywallComponent] }, + add: { imports: [StubPaywallComponent] }, + }); + create({}, 'paywall'); + const paywall = fixture.debugElement.query( + By.directive(StubPaywallComponent) + ); + expect(paywall).toBeTruthy(); + expect(paywall.componentInstance.serviceName).toBe('Compliance'); + expect(el().querySelector('.cp')).toBeNull(); + expect(api['list']).not.toHaveBeenCalled(); + }); + + it('offers retry when the list fails', () => { + api['list'].mockReturnValueOnce(throwError(() => ({ status: 500 }))); + create(); + expect(q('cp-load-failed')).toBeTruthy(); + }); +}); diff --git a/src/app/adf-compliance/df-compliance/df-compliance.component.ts b/src/app/adf-compliance/df-compliance/df-compliance.component.ts new file mode 100644 index 000000000..cda4aeb58 --- /dev/null +++ b/src/app/adf-compliance/df-compliance/df-compliance.component.ts @@ -0,0 +1,283 @@ +import { Component, OnDestroy, OnInit, inject } from '@angular/core'; +import { NgClass, NgFor, NgIf, NgTemplateOutlet } from '@angular/common'; +import { ActivatedRoute, Router, RouterLink } from '@angular/router'; +import { MatButtonModule } from '@angular/material/button'; +import { MatDialog, MatDialogModule } from '@angular/material/dialog'; +import { MatIconModule } from '@angular/material/icon'; +import { MatMenuModule } from '@angular/material/menu'; +import { MatProgressBarModule } from '@angular/material/progress-bar'; +import { MatSelectModule } from '@angular/material/select'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { TranslocoPipe } from '@ngneat/transloco'; +import { Subscription } from 'rxjs'; +import { DfEmptyStateComponent } from 'src/app/shared/components/df-empty-state/df-empty-state.component'; +import { DfPaywallComponent } from 'src/app/shared/components/df-paywall/df-paywall.component'; +import { DfSkeletonComponent } from 'src/app/shared/components/df-skeleton/df-skeleton.component'; +import { ComplianceStore, fixLink, scoreOf } from '../compliance.store'; +import { + BaselineControl, + BaselineDetail, + BaselineSummary, + ControlStatus, + ReportFormat, +} from '../compliance.types'; +import { + ApplyDialogData, + ApplyDialogResult, + DfComplianceApplyDialogComponent, +} from './df-compliance-apply-dialog.component'; +import { shortWhen } from 'src/app/adf-data-masking/dm-time'; + +/** The check that only "Apply with masking turned on" fixes. */ +const MODE_CHECK = 'masking.mode'; + +/** One action for a failing check: a page that fixes it, or Apply. */ +export type CheckAction = + | { kind: 'link'; path: string; query: Record } + | { kind: 'apply'; turnOn: boolean }; + +const STATUS_ICON: Record = { + pass: 'check_circle', + fail: 'cancel', + manual: 'radio_button_unchecked', + na: 'remove_circle_outline', +}; + +/** + * Compliance (/api-security/compliance): baselines grouped by vertical in + * a rail, the selected one's score, its checks (status, title, framework + * refs, one action) and Apply / Export / Stop tracking. Manual checks sit + * collapsed at the end. The selection lives in ?baseline= so links from + * Data Masking's first run (and reloads) land on it. + */ +@Component({ + selector: 'df-compliance', + standalone: true, + imports: [ + NgIf, + NgFor, + NgClass, + NgTemplateOutlet, + RouterLink, + MatButtonModule, + MatDialogModule, + MatIconModule, + MatMenuModule, + MatProgressBarModule, + MatSelectModule, + MatTooltipModule, + TranslocoPipe, + DfEmptyStateComponent, + DfPaywallComponent, + DfSkeletonComponent, + ], + providers: [ComplianceStore], + templateUrl: './df-compliance.component.html', + styleUrls: ['./df-compliance.component.scss'], +}) +export class DfComplianceComponent implements OnInit, OnDestroy { + readonly store = inject(ComplianceStore); + private route = inject(ActivatedRoute); + private router = inject(Router); + private dialog = inject(MatDialog); + + /** No security_baseline resource: the shared paywall instead. */ + paywall = false; + /** Manual checks start collapsed; reset when the baseline changes. */ + manualOpen = false; + private subs: Subscription[] = []; + + ngOnInit(): void { + this.subs.push( + this.route.data.subscribe(d => { + this.paywall = d?.['data'] === 'paywall'; + if (!this.paywall && !this.store.baselines) { + this.store.load(this.route.snapshot.queryParamMap.get('baseline')); + } + }), + this.route.queryParamMap.subscribe(q => { + const id = q.get('baseline'); + if (id && id !== this.store.selectedId && this.store.baselines) { + this.manualOpen = false; + this.store.select(id); + } + }) + ); + } + + ngOnDestroy(): void { + this.subs.forEach(s => s.unsubscribe()); + } + + select(id: string): void { + if (id === this.store.selectedId) return; + this.manualOpen = false; + this.store.select(id); + this.router.navigate([], { + relativeTo: this.route, + queryParams: { baseline: id }, + queryParamsHandling: 'merge', + replaceUrl: true, + }); + } + + // -------------------------------------------------------------- derived + + verticalLabel(v: string): string { + const key = `compliance.verticals.${v}`; + const label = this.store.t(`verticals.${v}`); + return label === key ? v : label; + } + + railScore(b: BaselineSummary): string { + return `${b.score.pass}/${b.score.pass + b.score.fail}`; + } + + railLabel(b: BaselineSummary): string { + const score = this.store.t('scoreLabel', { + pass: b.score.pass, + total: b.score.pass + b.score.fail, + }); + return b.adopted + ? `${b.name}, ${this.store.t('tracking')}, ${score}` + : `${b.name}, ${score}`; + } + + /** Checks the score counts, in server order; n/a ones trail, muted. */ + automated(d: BaselineDetail): BaselineControl[] { + const live = d.controls.filter( + c => c.status === 'pass' || c.status === 'fail' + ); + const na = d.controls.filter(c => c.status === 'na'); + return [...live, ...na]; + } + + manual(d: BaselineDetail): BaselineControl[] { + return d.controls.filter(c => c.status === 'manual'); + } + + score(d: BaselineDetail): { pass: number; checked: number; pct: number } { + const s = scoreOf(d.controls); + return { ...s, pct: s.checked ? (s.pass / s.checked) * 100 : 0 }; + } + + /** + * What was found, under the title: for a failing, manual or n/a check. + * A passing check's detail only restates its title, so it stays out + * (the evidence export carries every detail). + */ + showDetail(c: BaselineControl): boolean { + return !!c.detail && c.status !== 'pass'; + } + + statusIcon(c: BaselineControl): string { + return STATUS_ICON[c.status] ?? 'help_outline'; + } + + statusLabel(c: BaselineControl): string { + return this.store.t(`status.${c.status}`); + } + + /** The baseline's name as a title: "{{name}} baseline", never a score. */ + title(d: BaselineDetail): string { + return this.store.t('title', { name: d.name }); + } + + /** What it maps to, and that it is not a certification: one muted line. */ + mapped(d: BaselineDetail): string { + const note = this.store.t('notCertification'); + return d.frameworks?.length + ? `${this.store.t('mapped', { frameworks: d.frameworks.join(', ') })} · ${note}` + : note; + } + + /** + * A failing check's one action, or null. 'link': the page that fixes it. + * 'apply': Apply fixes it; "Masking is on" needs Apply with masking + * turned on, so its Fix opens Apply with that ticked. Before the first + * scan masking can't be turned on: that Fix goes to Data Masking, where + * the scan starts. + */ + action(c: BaselineControl): CheckAction | null { + if (c.status !== 'fail') return null; + if (c.fix?.kind === 'link') { + const l = fixLink(c.fix.route); + return l ? { kind: 'link', ...l } : null; + } + if (c.fix?.kind !== 'apply') return null; + if (c.id !== MODE_CHECK) return { kind: 'apply', turnOn: false }; + if (this.store.offerTurnOn) return { kind: 'apply', turnOn: true }; + return this.store.maskingMode === 'on' + ? null + : { kind: 'link', path: '/api-security/data-masking', query: {} }; + } + + trackingTip(d: BaselineDetail): string { + return d.adoptedAt + ? this.store.t('trackingSince', { when: shortWhen(d.adoptedAt) }) + : this.store.t('tracking'); + } + + /** + * Apply does something: adopt, change settings, or fix a failing check + * that Apply fixes (masking.mode needs it with masking turned on, which + * `changes` leaves out). Never hidden while such a check fails. + */ + canApply(d: BaselineDetail): boolean { + return ( + !d.adopted || + d.changes.length > 0 || + d.controls.some(c => c.status === 'fail' && c.fix?.kind === 'apply') + ); + } + + // -------------------------------------------------------------- actions + + /** Run a failing check's Apply action. */ + fixWithApply(d: BaselineDetail, a: CheckAction): void { + if (a.kind === 'apply') this.apply(d, a.turnOn); + } + + apply(d: BaselineDetail, turnOn = false): void { + if (this.store.busy) return; + const offerTurnOn = this.store.offerTurnOn; + const data: ApplyDialogData = { + name: d.name, + changes: d.changes, + offerTurnOn, + turnOn: offerTurnOn && turnOn, + changesOn$: offerTurnOn ? this.store.changesWithModeOn(d.id) : undefined, + }; + this.dialog + .open< + DfComplianceApplyDialogComponent, + ApplyDialogData, + ApplyDialogResult | undefined + >(DfComplianceApplyDialogComponent, { + data, + autoFocus: 'dialog', + width: '520px', + maxWidth: 'calc(100vw - 32px)', + }) + .afterClosed() + .subscribe(res => { + if (!res) return; + this.store + .apply(d.id, res.turnOn) + .subscribe({ error: () => undefined }); + }); + } + + stop(d: BaselineDetail): void { + this.store.stopTracking(d.id); + } + + exportReport(d: BaselineDetail, format: ReportFormat): void { + this.store.exportReport(d.id, format); + } + + trackGroup = (_: number, g: { vertical: string }) => g.vertical; + trackBaseline = (_: number, b: BaselineSummary) => b.id; + trackControl = (_: number, c: BaselineControl) => c.id; + trackRef = (_: number, r: string) => r; +} diff --git a/src/app/adf-data-explorer/df-data-grid.component.ts b/src/app/adf-data-explorer/df-data-grid.component.ts index 53d7cfa42..5eb67d13a 100644 --- a/src/app/adf-data-explorer/df-data-grid.component.ts +++ b/src/app/adf-data-explorer/df-data-grid.component.ts @@ -52,6 +52,7 @@ import { DfSchemaInfoComponent } from './df-schema-info.component'; import { DfRowDetailComponent } from './df-row-detail.component'; import { normalizeError } from 'src/app/shared/utilities/app-error'; import { RouterLink } from '@angular/router'; +import { MaskingNote, MaskingNotice, maskingNoteFor } from './masking-notice'; import { DfPresentationService } from 'src/app/shared/services/df-presentation.service'; import { maskSecret } from 'src/app/shared/utilities/mask'; @@ -111,6 +112,17 @@ interface ColumnFilter { {{ t('dataExplorer.readOnly') }} + + + + {{ t('dataExplorer.masked') }} +
    @@ -202,6 +214,21 @@ interface ColumnFilter {
    + +
    + + {{ + maskingNoteText(t) + }} +
    +
    = {}; activeFilterCount = 0; + + /** + * Data Masking's headers on the last response (masked values, ignored + * filters/sorts); null when there were none, which is always the case + * without the commercial df-masking package. + */ + maskingNotice: MaskingNotice | null = null; + /** The one-line note those headers call for, or null. */ + maskingNote: MaskingNote | null = null; private filterSubject$ = new Subject(); // Column resize state @@ -1537,6 +1627,8 @@ export class DfDataGridComponent this.cachedSchema = null; this.columnWidths = {}; this.navigationFilter = this.initialFilter || null; + this.maskingNotice = null; + this.maskingNote = null; this.initialLoading = true; this.loadData(); this.loadSchemaForTable(); @@ -1595,10 +1687,14 @@ export class DfDataGridComponent } this.dataSource.data = records; this.totalRecords = res.meta?.count ?? records.length; + this.maskingNotice = res.masking ?? null; + this.maskingNote = maskingNoteFor(this.maskingNotice); this.loading = false; this.initialLoading = false; }, error: err => { + this.maskingNotice = null; + this.maskingNote = null; this.error = normalizeError(err).message; this.loading = false; this.initialLoading = false; @@ -1838,6 +1934,32 @@ export class DfDataGridComponent this.loadData(); } + /** + * "Masked columns can't be filtered: email · Sort ignored on masked + * columns: ssn", from the note (`t` is the template's translate). + */ + maskingNoteText( + t: (key: string, params?: Record) => string + ): string { + const note = this.maskingNote; + if (!note) return ''; + const parts: string[] = []; + if (note.filter) { + parts.push( + note.filter.length + ? t('dataExplorer.maskingFilter', { cols: note.filter.join(', ') }) + : t('dataExplorer.maskingFilterAll') + ); + } + if (note.sort) { + parts.push(t('dataExplorer.maskingSort', { cols: note.sort.join(', ') })); + } + const line = parts.join(' · '); + return note.more + ? `${line} ${t('dataExplorer.maskingMore', { count: note.more })}` + : line; + } + private updateActiveFilterCount(): void { // Called after every columnFilters edit; also invalidates the API URL cache. this.apiStateVersion++; diff --git a/src/app/adf-data-explorer/df-data-grid.masking.spec.ts b/src/app/adf-data-explorer/df-data-grid.masking.spec.ts new file mode 100644 index 000000000..28f5351d3 --- /dev/null +++ b/src/app/adf-data-explorer/df-data-grid.masking.spec.ts @@ -0,0 +1,251 @@ +/** + * Data Explorer and Data Masking (df-masking): a small "Masked" marker when + * the response carries X-DreamFactory-Masking: on, one line naming the + * masked columns a filter could not use (they match no rows, decision 20) + * or a sort ignored (X-DreamFactory-Masking-Dropped), and nothing at all + * when the headers are absent (OSS, unmasked use). + */ +import { + ComponentFixture, + TestBed, + fakeAsync, + tick, +} from '@angular/core/testing'; +import { HttpHeaders, provideHttpClient } from '@angular/common/http'; +import { + HttpTestingController, + provideHttpClientTesting, +} from '@angular/common/http/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { provideRouter } from '@angular/router'; +import { SimpleChange } from '@angular/core'; +import { Translation, TranslocoTestingModule } from '@ngneat/transloco'; +import { of } from 'rxjs'; +import { DfDataGridComponent } from './df-data-grid.component'; +import { + DataExplorerService, + TableDataResponse, +} from './services/data-explorer.service'; +import { readMaskingNotice } from './masking-notice'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const DE: Translation = require('../../assets/i18n/dataExplorer/en.json'); + +const ROWS = [ + { id: 1, email: 'EMAIL_7f3a', last_name: 'NAME_19c2' }, + { id: 2, email: 'EMAIL_0b9d', last_name: 'NAME_44e1' }, +]; + +describe('DfDataGridComponent and Data Masking headers', () => { + let fixture: ComponentFixture; + let api: Record; + let response: TableDataResponse; + + const el = () => fixture.nativeElement as HTMLElement; + const q = (id: string) => + el().querySelector(`[data-testid="${id}"]`) as HTMLElement | null; + const text = (e: Element | null) => + e?.textContent?.replace(/\s+/g, ' ').trim(); + + function create() { + TestBed.configureTestingModule({ + imports: [ + DfDataGridComponent, + NoopAnimationsModule, + TranslocoTestingModule.forRoot({ + langs: { en: {}, 'dataExplorer/en': DE }, + translocoConfig: { availableLangs: ['en'], defaultLang: 'en' }, + preloadLangs: true, + }), + ], + providers: [ + provideRouter([]), + { provide: DataExplorerService, useValue: api }, + ], + }); + fixture = TestBed.createComponent(DfDataGridComponent); + const c = fixture.componentInstance; + c.serviceName = 'crm_mysql'; + c.tableName = 'customers'; + c.ngOnChanges({ tableName: new SimpleChange(null, 'customers', true) }); + fixture.detectChanges(); + } + + beforeEach(() => { + response = { resource: ROWS, meta: { count: 300 }, masking: null }; + api = { + getTableData: jest.fn(() => of(response)), + getTableSchema: jest.fn(() => + of({ name: 'customers', field: [], related: [] }) + ), + findReadKeys: jest.fn(() => of([])), + }; + }); + + it('looks exactly as before when the response has no masking headers', () => { + create(); + expect(el().querySelector('.data-table')).toBeTruthy(); + expect(q('de-masked')).toBeNull(); + expect(q('de-masking-dropped')).toBeNull(); + expect(el().querySelector('.masked-chip, .masking-note')).toBeNull(); + }); + + it('shows a small Masked marker when values are masked', () => { + response.masking = readMaskingNotice( + new HttpHeaders({ 'X-DreamFactory-Masking': 'on' }) + ); + create(); + expect(text(q('de-masked'))).toBe('visibility_off Masked'); + expect(q('de-masked')!.getAttribute('href')).toBe( + '/api-security/data-masking?tab=access' + ); + expect(q('de-masking-dropped')).toBeNull(); + }); + + it('says in one line that masked columns cannot be filtered (no rows) and that a sort on one was ignored', fakeAsync(() => { + create(); + const c = fixture.componentInstance; + // Decision 20: the admin filters EMAIL (masked). The server treats the + // comparison as unknown, so it matches no rows; the sort on a masked + // column is ignored. + response = { + resource: [], + meta: { count: 0 }, + masking: readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking': 'on', + 'X-DreamFactory-Masking-Dropped': 'filter:email,order:last%20name', + }) + ), + }; + c.columnFilters = { email: { op: 'contains', value: 'zzzz' } }; + ( + c as unknown as { updateActiveFilterCount(): void } + ).updateActiveFilterCount(); + c.loadData(); + fixture.detectChanges(); + tick(); + const note = q('de-masking-dropped')!; + expect(note.querySelectorAll('.masking-note-text').length).toBe(1); + expect(text(note.querySelector('.masking-note-text'))).toBe( + "Masked columns can't be filtered: email · Sort ignored on masked columns: last name" + ); + expect( + note.querySelector('.masking-note-text')!.getAttribute('title') + ).toBe( + "Masked columns can't be filtered: email · Sort ignored on masked columns: last name" + ); + // The rows are filtered (to none): the footer says so, as without masking. + expect(text(el().querySelector('.grid-footer .footer-info'))).toBe( + '0 records (filtered)' + ); + + // email OR id = 1: the clear half still selects its row. + response = { + resource: [ROWS[0]], + meta: { count: 1 }, + masking: readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking': 'on', + 'X-DreamFactory-Masking-Dropped': 'filter:email', + }) + ), + }; + c.columnFilters = { + email: { op: 'contains', value: 'zzzz' }, + id: { op: 'eq', value: '1' }, + }; + ( + c as unknown as { updateActiveFilterCount(): void } + ).updateActiveFilterCount(); + c.loadData(); + fixture.detectChanges(); + expect(text(q('de-masking-dropped')!.querySelector('span'))).toBe( + "Masked columns can't be filtered: email" + ); + expect(text(el().querySelector('.grid-footer .footer-info'))).toBe( + '1 records (filtered)' + ); + })); + + it('a sort on a masked column alone: "Sort ignored on masked columns"', () => { + response.masking = readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking': 'on', + 'X-DreamFactory-Masking-Dropped': 'order:ssn', + }) + ); + create(); + expect(text(q('de-masking-dropped')!.querySelector('span'))).toBe( + 'Sort ignored on masked columns: ssn' + ); + }); + + it('clears the notice when the next response has no headers', () => { + response.masking = readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking': 'on', + 'X-DreamFactory-Masking-Dropped': 'filter:*,+2 more', + }) + ); + create(); + expect(text(q('de-masking-dropped')!.querySelector('span'))).toBe( + "Masked columns can't be filtered +2 more" + ); + response = { resource: ROWS, meta: { count: 300 }, masking: null }; + fixture.componentInstance.loadData(); + fixture.detectChanges(); + expect(q('de-masked')).toBeNull(); + expect(q('de-masking-dropped')).toBeNull(); + }); +}); + +describe('DataExplorerService.getTableData masking headers', () => { + it('passes the body through and reads the headers into `masking`', () => { + TestBed.resetTestingModule(); + TestBed.configureTestingModule({ + providers: [provideHttpClient(), provideHttpClientTesting()], + }); + const svc = TestBed.inject(DataExplorerService); + const http = TestBed.inject(HttpTestingController); + const got: TableDataResponse[] = []; + svc + .getTableData('crm_mysql', 'customers', 25, 0) + .subscribe(r => got.push(r)); + svc + .getTableData( + 'crm_mysql', + 'customers', + 25, + 0, + undefined, + '(email like "%z%")' + ) + .subscribe(r => got.push(r)); + const [plain, masked] = http.match(r => + r.url.endsWith('/crm_mysql/_table/customers') + ); + plain.flush({ resource: ROWS, meta: { count: 300 } }); + masked.flush( + { resource: ROWS, meta: { count: 300 } }, + { + headers: { + 'X-DreamFactory-Masking': 'on', + 'X-DreamFactory-Masking-Dropped': 'filter:email', + }, + } + ); + expect(got[0]).toEqual({ + resource: ROWS, + meta: { count: 300 }, + masking: null, + }); + expect(got[1].resource).toEqual(ROWS); + expect(got[1].masking).toEqual({ + masked: true, + dropped: [{ kind: 'filter', columns: ['email'], all: false }], + more: 0, + }); + http.verify(); + }); +}); diff --git a/src/app/adf-data-explorer/masking-notice.spec.ts b/src/app/adf-data-explorer/masking-notice.spec.ts new file mode 100644 index 000000000..3ce50a066 --- /dev/null +++ b/src/app/adf-data-explorer/masking-notice.spec.ts @@ -0,0 +1,102 @@ +import { HttpHeaders } from '@angular/common/http'; +import { maskingNoteFor, readMaskingNotice } from './masking-notice'; + +describe('readMaskingNotice', () => { + it('is null without masking headers (OSS, masking off, nothing masked)', () => { + expect(readMaskingNotice(new HttpHeaders())).toBeNull(); + expect(readMaskingNotice(null)).toBeNull(); + expect( + readMaskingNotice(new HttpHeaders({ 'Content-Type': 'application/json' })) + ).toBeNull(); + }); + + it('Preview masks nothing: no marker', () => { + expect( + readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking': 'preview', + 'X-DreamFactory-Masking-Would-Mask': 'email,ssn', + }) + ) + ).toBeNull(); + }); + + it('marks a masked response', () => { + expect( + readMaskingNotice(new HttpHeaders({ 'x-dreamfactory-masking': 'on' })) + ).toEqual({ masked: true, dropped: [], more: 0 }); + }); + + it('groups ignored parts by kind, decoding percent-encoded names', () => { + const n = readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking': 'on', + 'X-DreamFactory-Masking-Dropped': + 'filter:email,order:last%20name,having:ssn,filter:email,filter:pr%C3%A9nom,group:city,fields:dob', + }) + )!; + expect(n.dropped).toEqual([ + { kind: 'filter', columns: ['email', 'ssn', 'prénom'], all: false }, + { kind: 'order', columns: ['last name'], all: false }, + { kind: 'group', columns: ['city'], all: false }, + { kind: 'fields', columns: ['dob'], all: false }, + ]); + expect(n.more).toBe(0); + }); + + it('reads the whole-filter drop, the "+N more" cap, and keeps bad escapes raw', () => { + const n = readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking-Dropped': 'filter:*, order:a%zz ,+3 more', + }) + )!; + expect(n.masked).toBe(false); + expect(n.dropped).toEqual([ + { kind: 'filter', columns: [], all: true }, + { kind: 'order', columns: ['a%zz'], all: false }, + ]); + expect(n.more).toBe(3); + }); +}); + +describe('maskingNoteFor (decision 20: filters on masked columns match no rows)', () => { + const notice = (dropped: string) => + readMaskingNotice( + new HttpHeaders({ + 'X-DreamFactory-Masking': 'on', + 'X-DreamFactory-Masking-Dropped': dropped, + }) + ); + + it('nothing to say without the Dropped header (or without headers)', () => { + expect(maskingNoteFor(null)).toBeNull(); + expect( + maskingNoteFor( + readMaskingNotice(new HttpHeaders({ 'X-DreamFactory-Masking': 'on' })) + ) + ).toBeNull(); + }); + + it('filter and having columns read as "can\'t be filtered"; order and group as "sort ignored"', () => { + expect( + maskingNoteFor(notice('filter:email,having:ssn,order:last%20name')) + ).toEqual({ filter: ['email', 'ssn'], sort: ['last name'], more: 0 }); + expect(maskingNoteFor(notice('order:ssn,group:ssn,group:city'))).toEqual({ + filter: null, + sort: ['ssn', 'city'], + more: 0, + }); + }); + + it('the whole filter (*) has no column list; "+N more" is kept', () => { + expect(maskingNoteFor(notice('filter:*,+2 more'))).toEqual({ + filter: [], + sort: null, + more: 2, + }); + }); + + it('an aggregate item alone (fields:) has no line: Data Explorer never sends one', () => { + expect(maskingNoteFor(notice('fields:dob'))).toBeNull(); + }); +}); diff --git a/src/app/adf-data-explorer/masking-notice.ts b/src/app/adf-data-explorer/masking-notice.ts new file mode 100644 index 000000000..fd2539024 --- /dev/null +++ b/src/app/adf-data-explorer/masking-notice.ts @@ -0,0 +1,122 @@ +/** + * What df-masking (commercial Data Masking) said about a data response, read + * from its headers. Both are absent unless the package is installed, masking + * is on, and the response carries a masked column, so OSS and unmasked use + * never produce a notice. + * + * - `X-DreamFactory-Masking: on` - some values in this response are masked. + * - `X-DreamFactory-Masking-Dropped: filter:email,order:last%20name,+2 more` + * - request parts that met a masked column (column names only, + * percent-encoded per RFC 3986, `*` = the whole filter, capped with + * `,+N more`). `filter:
    ` means conditions on that column matched no + * rows: the server treats a comparison it may not evaluate as unknown + * (SQL NULL), so a filter never widens. `order:`/`group:` were ignored. + */ +import { HttpHeaders } from '@angular/common/http'; + +export const MASKING_HEADER = 'X-DreamFactory-Masking'; +export const MASKING_DROPPED_HEADER = 'X-DreamFactory-Masking-Dropped'; + +export type DroppedKind = 'filter' | 'order' | 'group' | 'fields'; + +export interface DroppedGroup { + kind: DroppedKind; + /** Column names, decoded, in header order. */ + columns: string[]; + /** The whole filter was ignored (it could not be checked safely). */ + all: boolean; +} + +export interface MaskingNotice { + /** Values in this response are masked. */ + masked: boolean; + /** Ignored request parts, one group per kind, in header order. */ + dropped: DroppedGroup[]; + /** Further ignored parts the header left out (`+N more`). */ + more: number; +} + +const KINDS: Record = { + filter: 'filter', + having: 'filter', + order: 'order', + group: 'group', + fields: 'fields', +}; + +const MORE = /^\+(\d+)\s*more$/i; + +function decode(name: string): string { + try { + return decodeURIComponent(name); + } catch { + return name; + } +} + +/** The notice for a response, or null when it carries no masking headers. */ +export function readMaskingNotice( + headers: HttpHeaders | null | undefined +): MaskingNotice | null { + if (!headers) return null; + const masked = headers.get(MASKING_HEADER)?.trim().toLowerCase() === 'on'; + const raw = headers.get(MASKING_DROPPED_HEADER)?.trim() ?? ''; + if (!masked && !raw) return null; + + const groups = new Map(); + let more = 0; + for (const part of raw.split(',')) { + const item = part.trim(); + if (!item) continue; + const m = MORE.exec(item); + if (m) { + more += Number(m[1]); + continue; + } + const colon = item.indexOf(':'); + const kind = + KINDS[colon > 0 ? item.slice(0, colon).trim().toLowerCase() : 'filter'] ?? + 'filter'; + const name = decode(colon > 0 ? item.slice(colon + 1).trim() : item); + let group = groups.get(kind); + if (!group) { + group = { kind, columns: [], all: false }; + groups.set(kind, group); + } + if (name === '*') group.all = true; + else if (name && !group.columns.includes(name)) group.columns.push(name); + } + return { masked, dropped: [...groups.values()], more }; +} + +/** What the Data Explorer note says: one line, built from the header. */ +export interface MaskingNote { + /** + * Masked columns a filter could not use (they matched no rows); empty + * when only the whole filter was named (`*`); null without filter items. + */ + filter: string[] | null; + /** Masked columns a sort or group by ignored; null without any. */ + sort: string[] | null; + /** Further items the header left out (`+N more`). */ + more: number; +} + +/** + * The note for a response, or null when there is nothing to say. Data + * Explorer sends no `fields` or `group`, so a `fields` item (an aggregate + * the server removed) has no line of its own; group by reads as a sort. + */ +export function maskingNoteFor( + notice: MaskingNotice | null | undefined +): MaskingNote | null { + if (!notice) return null; + const f = notice.dropped.find(g => g.kind === 'filter'); + const filter = f && (f.all || f.columns.length) ? f.columns : null; + const sorted = notice.dropped + .filter(g => g.kind === 'order' || g.kind === 'group') + .flatMap(g => g.columns); + const sort = sorted.length ? [...new Set(sorted)] : null; + if (!filter && !sort) return null; + return { filter, sort, more: notice.more }; +} diff --git a/src/app/adf-data-explorer/services/data-explorer.service.ts b/src/app/adf-data-explorer/services/data-explorer.service.ts index d9b298ecc..f347ff978 100644 --- a/src/app/adf-data-explorer/services/data-explorer.service.ts +++ b/src/app/adf-data-explorer/services/data-explorer.service.ts @@ -4,6 +4,7 @@ import { Observable, forkJoin, of } from 'rxjs'; import { map, switchMap } from 'rxjs/operators'; import { BASE_URL } from '../../shared/constants/urls'; import { toastOff } from '../../shared/utilities/http-contexts'; +import { MaskingNotice, readMaskingNotice } from '../masking-notice'; // toast-off on every request: the data explorer components render their own // in-page error states (with Retry), so the interceptor must not also toast. @@ -39,6 +40,11 @@ export interface TableDataResponse { meta?: { count: number; }; + /** + * From the response headers, not the body: what Data Masking (df-masking) + * did to this response. Null when it sent no masking headers. + */ + masking?: MaskingNotice | null; } export interface FieldInfo { @@ -208,14 +214,22 @@ export class DataExplorerService { if (filter) { params.filter = filter; } - return this.http.get( - `${BASE_URL}/${serviceName}/_table/${tableName}`, - { - params, - headers: { 'show-loading': '' }, - context: toastOff(), - } - ); + return this.http + .get( + `${BASE_URL}/${serviceName}/_table/${tableName}`, + { + params, + headers: { 'show-loading': '' }, + context: toastOff(), + observe: 'response', + } + ) + .pipe( + map(res => ({ + ...(res.body ?? { resource: [] }), + masking: readMaskingNotice(res.headers), + })) + ); } /** diff --git a/src/app/adf-data-masking/_dm-controls.scss b/src/app/adf-data-masking/_dm-controls.scss new file mode 100644 index 000000000..ef9388f88 --- /dev/null +++ b/src/app/adf-data-masking/_dm-controls.scss @@ -0,0 +1,86 @@ +// Compact standalone mat-select (no form-field chrome) for dense rows. +@mixin dm-select { + .dm-select { + background: var(--df-surface); + box-sizing: border-box; + border: 1px solid var(--df-border); + border-radius: var(--df-radius-sm); + font-size: var(--df-font-size-sm); + height: 30px; + line-height: 28px; + padding: 0 var(--df-space-2) 0 var(--df-space-3); + width: 176px; + + &:hover:not(.mat-mdc-select-disabled) { + border-color: var(--df-text-faint); + } + + &.mat-focused { + border-color: var(--df-accent); + box-shadow: 0 0 0 1px var(--df-accent); + } + + &.mat-mdc-select-disabled { + opacity: 0.6; + } + + ::ng-deep .mat-mdc-select-trigger { + height: 28px; + } + + ::ng-deep .mat-mdc-select-value, + ::ng-deep .mat-mdc-select-value-text { + color: var(--df-text); + } + + &.mat-mdc-select-disabled ::ng-deep .mat-mdc-select-value, + &.mat-mdc-select-disabled ::ng-deep .mat-mdc-select-value-text { + color: var(--df-text-muted); + } + + ::ng-deep .mat-mdc-select-placeholder { + color: var(--df-text-faint); + } + } +} + +// Compact mat-button-toggle segmented control. +@mixin dm-segmented { + .dm-seg { + --mat-standard-button-toggle-height: 30px; + font-size: var(--df-font-size-sm); + + ::ng-deep .mat-button-toggle-label-content { + line-height: 28px !important; + padding: 0 var(--df-space-3) !important; + } + + ::ng-deep .mat-button-toggle-checked { + font-weight: var(--df-font-weight-medium); + } + } +} + +// The one-line lock note ("Only the super admin can …"): same look on the +// mode bar and the Access tab. +@mixin dm-lock { + .lock { + align-items: center; + background: var(--df-surface-2); + border-bottom: 1px solid var(--df-border-2); + color: var(--df-text-2); + display: flex; + font-size: var(--df-font-size-sm); + gap: var(--df-space-2); + margin: 0; + padding: var(--df-space-2) var(--df-space-3); + + mat-icon { + color: var(--df-text-muted); + flex: none; + font-size: 16px; + height: 16px; + width: 16px; + } + } +} diff --git a/src/app/adf-data-masking/_dm-tier.scss b/src/app/adf-data-masking/_dm-tier.scss new file mode 100644 index 000000000..3359375e4 --- /dev/null +++ b/src/app/adf-data-masking/_dm-tier.scss @@ -0,0 +1,44 @@ +// Data Masking tier hooks. `.dm-tier-` sets --dm-tier / --dm-tier-soft +// from the --df-mask-* tokens (styles.scss, all three themes) so chips, +// bars, dots and strip ticks read one variable. --dm-tier-fill is what solid +// marks (bars, ticks, meters) paint with: the tier colour, unless a theme +// gives the tier a texture (phosphor hatches identity, since its tiers +// cannot part by hue). --dm-tier-ink is the tier colour for small text on +// its own wash: identity's amber is mixed toward the theme's text colour +// (darker in light, lighter in dark) so 12px labels keep 4.5:1. +@mixin dm-tiers { + .dm-tier-restricted { + --dm-tier: var(--df-mask-restricted); + --dm-tier-soft: var(--df-mask-restricted-soft); + --dm-tier-fill: var(--df-mask-restricted-fill, var(--df-mask-restricted)); + --dm-tier-ink: var(--df-mask-restricted); + } + .dm-tier-identity { + --dm-tier: var(--df-mask-identity); + --dm-tier-soft: var(--df-mask-identity-soft); + --dm-tier-fill: var(--df-mask-identity-fill, var(--df-mask-identity)); + --dm-tier-ink: color-mix( + in srgb, + var(--df-mask-identity) 76%, + var(--df-text) + ); + } + .dm-tier-quasi { + --dm-tier: var(--df-mask-quasi); + --dm-tier-soft: var(--df-mask-quasi-soft); + --dm-tier-fill: var(--df-mask-quasi-fill, var(--df-mask-quasi)); + --dm-tier-ink: var(--df-mask-quasi); + } + .dm-tier-free_text { + --dm-tier: var(--df-mask-free-text); + --dm-tier-soft: var(--df-mask-free-text-soft); + --dm-tier-fill: var(--df-mask-free-text-fill, var(--df-mask-free-text)); + --dm-tier-ink: var(--df-mask-free-text); + } + .dm-tier-clear { + --dm-tier: var(--df-text-faint); + --dm-tier-soft: var(--df-surface-2); + --dm-tier-fill: var(--df-text-faint); + --dm-tier-ink: var(--df-text-muted); + } +} diff --git a/src/app/adf-data-masking/data-masking-api.service.spec.ts b/src/app/adf-data-masking/data-masking-api.service.spec.ts new file mode 100644 index 000000000..6a9ae0f38 --- /dev/null +++ b/src/app/adf-data-masking/data-masking-api.service.spec.ts @@ -0,0 +1,154 @@ +import { TestBed } from '@angular/core/testing'; +import { + HttpClientTestingModule, + HttpTestingController, +} from '@angular/common/http/testing'; +import { + DATA_MASKING_URL, + DfDataMaskingApiService, +} from './data-masking-api.service'; +import { ERROR_HANDLING } from 'src/app/shared/utilities/http-contexts'; + +describe('DfDataMaskingApiService', () => { + let api: DfDataMaskingApiService; + let http: HttpTestingController; + + beforeEach(() => { + TestBed.configureTestingModule({ imports: [HttpClientTestingModule] }); + api = TestBed.inject(DfDataMaskingApiService); + http = TestBed.inject(HttpTestingController); + }); + + afterEach(() => http.verify()); + + it('reads the overview from the system resource', () => { + api.overview().subscribe(); + const req = http.expectOne(DATA_MASKING_URL); + expect(DATA_MASKING_URL).toBe('/api/v2/system/data_masking'); + expect(req.request.method).toBe('GET'); + req.flush({}); + }); + + it('can read the overview quietly (no error toast)', () => { + api.overview(true).subscribe(); + const req = http.expectOne(DATA_MASKING_URL); + expect(req.request.context.get(ERROR_HANDLING)).toBe('silent'); + req.flush({}); + }); + + it('patches the mode with toast-off error handling', () => { + api.setMode('on').subscribe(); + const req = http.expectOne(DATA_MASKING_URL); + expect(req.request.method).toBe('PATCH'); + expect(req.request.body).toEqual({ mode: 'on' }); + expect(req.request.context.get(ERROR_HANDLING)).toBe('toast-off'); + req.flush({}); + }); + + it('lists columns per service and unwraps resource', done => { + api.columns('crm').subscribe(cols => { + expect(cols.length).toBe(1); + done(); + }); + const req = http.expectOne(r => r.url === `${DATA_MASKING_URL}/column`); + expect(req.request.params.get('service')).toBe('crm'); + req.flush({ resource: [{ id: 'crm.a.b' }] }); + }); + + it('sends the column id in the body, so any column name works', () => { + // A name a route segment cannot carry: slash, percent, non-Latin. + const id = 'crm.survey/responses.e mail%ł'; + api.decide(id, null).subscribe(); + const req = http.expectOne(`${DATA_MASKING_URL}/column`); + expect(req.request.method).toBe('PATCH'); + expect(req.request.body).toEqual({ id, class: null }); + expect(req.request.context.get(ERROR_HANDLING)).toBe('toast-off'); + req.flush({}); + }); + + it('counts the tables a database lists, quietly', done => { + api.tableCount('db').subscribe(n => { + expect(n).toBe(0); + done(); + }); + const req = http.expectOne(r => r.url.endsWith('/db/_table')); + expect(req.request.params.get('as_list')).toBe('true'); + expect(req.request.context.get(ERROR_HANDLING)).toBe('silent'); + req.flush({ resource: [] }); + }); + + it('tests a rule pattern for the live preview', () => { + api.testRule({ match: 'name', pattern: '^ssn$' }).subscribe(); + const req = http.expectOne(`${DATA_MASKING_URL}/rule/test`); + expect(req.request.method).toBe('POST'); + expect(req.request.body).toEqual({ match: 'name', pattern: '^ssn$' }); + req.flush({ matches: [] }); + }); + + it('patches a mask format as a resource list', () => { + api.setMask('email', 'initial').subscribe(); + const req = http.expectOne(`${DATA_MASKING_URL}/mask`); + expect(req.request.body).toEqual({ + resource: [{ class: 'email', format: 'initial' }], + }); + req.flush({ resource: [] }); + }); + + it('starts a scan of everything, or of one service', () => { + api.scan().subscribe(); + let req = http.expectOne(`${DATA_MASKING_URL}/scan`); + expect(req.request.method).toBe('POST'); + expect(req.request.body).toEqual({}); + req.flush({}); + api.scan('hr').subscribe(); + req = http.expectOne(`${DATA_MASKING_URL}/scan`); + expect(req.request.body).toEqual({ service: 'hr' }); + req.flush({}); + }); + + it('saves access as the complete grant list (camelCase in, snake on the wire)', () => { + api + .updateAccess({ admins: { grants: [{ adminId: 2, duration: '1h' }] } }) + .subscribe(); + const req = http.expectOne(`${DATA_MASKING_URL}/access`); + expect(req.request.method).toBe('PATCH'); + // The case interceptor (not under test here) snake_cases the body. + expect(req.request.body).toEqual({ + admins: { grants: [{ adminId: 2, duration: '1h' }] }, + }); + req.flush({}); + }); + + it('reads activity newest first as a resource list', done => { + api.activity().subscribe(a => { + expect(a.events.map(e => e.kind)).toEqual(['baseline', 'preview']); + expect(a.hasMore).toBe(false); + done(); + }); + http + .expectOne(`${DATA_MASKING_URL}/activity`) + .flush({ resource: [{ kind: 'baseline' }, { kind: 'preview' }] }); + }); + + it('asks the server for kinds, a page size and older rows', done => { + api + .activity({ kinds: ['mode', 'rule'], limit: 200, before: 41 }) + .subscribe(a => { + expect(a.hasMore).toBe(true); + done(); + }); + const req = http.expectOne(r => r.url === `${DATA_MASKING_URL}/activity`); + expect(req.request.params.get('kind')).toBe('mode,rule'); + expect(req.request.params.get('limit')).toBe('200'); + expect(req.request.params.get('before')).toBe('41'); + // The case interceptor (not under test) camelCases has_more. + req.flush({ resource: [], hasMore: true }); + }); + + it('deletes a rule by id', () => { + api.deleteRule(7).subscribe(); + const req = http.expectOne(`${DATA_MASKING_URL}/rule/7`); + expect(req.request.method).toBe('DELETE'); + req.flush({}); + }); +}); diff --git a/src/app/adf-data-masking/data-masking-api.service.ts b/src/app/adf-data-masking/data-masking-api.service.ts new file mode 100644 index 000000000..70579b606 --- /dev/null +++ b/src/app/adf-data-masking/data-masking-api.service.ts @@ -0,0 +1,201 @@ +/** + * HTTP for /api/v2/system/data_masking (backend contract, section 8). Wire + * keys are snake_case: the case interceptor camelCases responses and + * snake_cases request bodies, so this file speaks camelCase only. Data + * (class ids, column names, formats) is never used as an object key. + * + * Reads use the default error + * handling (normalized + toasted by the error interceptor) and callers keep + * whatever they last had, so a failed call never blanks the page. Writes + * are toast-off: the store shows one toast and rolls back. + */ +import { HttpClient, HttpParams } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Observable, map } from 'rxjs'; +import { BASE_URL } from 'src/app/shared/constants/urls'; +import { silent, toastOff } from 'src/app/shared/utilities/http-contexts'; +import { + AccessPatch, + AccessSettings, + ActivityEvent, + ActivityPage, + ActivityQuery, + BaselineSummary, + BuiltInRules, + DetectionRule, + MaskSetting, + MaskedColumn, + MaskingMode, + MaskingOverview, + RuleTestResult, + ServiceTables, +} from './data-masking.types'; +import { MaskFormat } from './data-masking.catalog'; + +export const DATA_MASKING_URL = `${BASE_URL}/system/data_masking`; + +interface ListResponse { + resource: T[]; + /** Activity: older events exist (has_more on the wire). */ + hasMore?: boolean; +} + +@Injectable({ providedIn: 'root' }) +export class DfDataMaskingApiService { + constructor(private http: HttpClient) {} + + /** + * `quiet` skips the error toast, for callers that only peek at the + * overview (Compliance reads the mode) and degrade on their own. + */ + overview(quiet = false): Observable { + return this.http.get( + DATA_MASKING_URL, + quiet ? { context: silent() } : {} + ); + } + + setMode(mode: MaskingMode): Observable { + return this.http.patch( + DATA_MASKING_URL, + { mode }, + { context: toastOff() } + ); + } + + tables(): Observable { + return this.http + .get>(`${DATA_MASKING_URL}/table`) + .pipe(map(r => r?.resource ?? [])); + } + + columns(service: string): Observable { + return this.http + .get>(`${DATA_MASKING_URL}/column`, { + params: { service }, + }) + .pipe(map(r => r?.resource ?? [])); + } + + /** + * null = not sensitive; a class id = masked with that class. The id goes + * in the body: column names can hold characters a route segment cannot. + */ + decide(id: string, cls: string | null): Observable { + return this.http.patch( + `${DATA_MASKING_URL}/column`, + { id, class: cls }, + { context: toastOff() } + ); + } + + /** + * How many tables a database service lists (DreamFactory's own _table + * resource). First run uses it to tell an empty database from one worth + * scanning; it is quiet because the caller degrades on a failure. + */ + tableCount(service: string): Observable { + return this.http + .get< + ListResponse + >(`${BASE_URL}/${encodeURIComponent(service)}/_table`, { params: { as_list: true }, context: silent() }) + .pipe(map(r => (Array.isArray(r?.resource) ? r.resource.length : 0))); + } + + /** Compliance baselines and whether each is tracked (quiet). */ + baselines(): Observable { + return this.http + .get>( + `${BASE_URL}/system/security_baseline`, + { + context: silent(), + } + ) + .pipe(map(r => r?.resource ?? [])); + } + + scan(service?: string): Observable { + return this.http.post( + `${DATA_MASKING_URL}/scan`, + service ? { service } : {}, + { context: toastOff() } + ); + } + + /** `quiet` for background re-reads that keep what is on screen. */ + access(quiet = false): Observable { + return this.http.get( + `${DATA_MASKING_URL}/access`, + quiet ? { context: silent() } : {} + ); + } + + updateAccess(patch: AccessPatch): Observable { + return this.http.patch( + `${DATA_MASKING_URL}/access`, + patch, + { context: toastOff() } + ); + } + + masks(): Observable { + return this.http + .get>(`${DATA_MASKING_URL}/mask`) + .pipe(map(r => r?.resource ?? [])); + } + + setMask(cls: string, format: MaskFormat): Observable { + return this.http + .patch< + ListResponse + >(`${DATA_MASKING_URL}/mask`, { resource: [{ class: cls, format }] }, { context: toastOff() }) + .pipe(map(r => r?.resource ?? [])); + } + + rules(): Observable { + return this.http + .get>(`${DATA_MASKING_URL}/rule`) + .pipe(map(r => r?.resource ?? [])); + } + + builtInRules(): Observable { + return this.http.get(`${DATA_MASKING_URL}/rule/builtin`); + } + + addRule( + rule: Pick + ): Observable { + return this.http.post(`${DATA_MASKING_URL}/rule`, rule, { + context: toastOff(), + }); + } + + deleteRule(id: number): Observable { + return this.http.delete(`${DATA_MASKING_URL}/rule/${id}`, { + context: toastOff(), + }); + } + + testRule( + rule: Pick + ): Observable { + return this.http.post( + `${DATA_MASKING_URL}/rule/test`, + rule, + { context: toastOff() } + ); + } + + /** One page of activity, filtered and paged by the server. */ + activity(q: ActivityQuery = {}): Observable { + let params = new HttpParams(); + if (q.kinds?.length) params = params.set('kind', q.kinds.join(',')); + if (q.limit) params = params.set('limit', q.limit); + if (q.before) params = params.set('before', q.before); + return this.http + .get>(`${DATA_MASKING_URL}/activity`, { + params, + }) + .pipe(map(r => ({ events: r?.resource ?? [], hasMore: !!r?.hasMore }))); + } +} diff --git a/src/app/adf-data-masking/data-masking.catalog.ts b/src/app/adf-data-masking/data-masking.catalog.ts new file mode 100644 index 000000000..02e7b0380 --- /dev/null +++ b/src/app/adf-data-masking/data-masking.catalog.ts @@ -0,0 +1,274 @@ +/** + * Data Masking catalog: the fixed class ids, their tiers, mask formats and a + * synthetic sample per class. Pure TypeScript with no Angular imports so the + * Playwright mock and the fixture generator can share it with the UI. + * + * Labels live in i18n (dataMasking/en.json, keys `classes.`, + * `tiers.`, `formats.`); this file only holds ids and behavior. + */ + +export type Tier = 'restricted' | 'identity' | 'quasi' | 'free_text'; + +export type MaskFormat = + | 'redact' + | 'last4' + | 'initial' + | 'year' + | 'token' + | 'null' + | 'span'; + +export interface ClassDef { + id: string; + tier: Tier; + /** Format applied when an admin has not chosen one. */ + format: MaskFormat; + /** Formats offered on the Masks tab. Restricted never offers token. */ + formats: MaskFormat[]; + /** Synthetic, obviously fake sample for the Masks tab before -> after. */ + sample: string; + /** Prefix for stable typed tokens (EMAIL_7f3a). */ + token: string; +} + +export const TIERS: Tier[] = ['restricted', 'identity', 'quasi', 'free_text']; + +export const CLASSES: ClassDef[] = [ + // restricted + { + id: 'national_id', + tier: 'restricted', + format: 'redact', + formats: ['redact', 'last4', 'null'], + sample: '123-45-6789', + token: 'NID', + }, + { + id: 'tax_id', + tier: 'restricted', + format: 'redact', + formats: ['redact', 'last4', 'null'], + sample: '12-3456789', + token: 'TAX', + }, + { + id: 'passport_number', + tier: 'restricted', + format: 'redact', + formats: ['redact', 'last4', 'null'], + sample: 'X12345678', + token: 'PASS', + }, + { + id: 'drivers_license', + tier: 'restricted', + format: 'redact', + formats: ['redact', 'last4', 'null'], + sample: 'D123-4567-8901', + token: 'DL', + }, + { + id: 'card_number', + tier: 'restricted', + format: 'last4', + formats: ['last4', 'redact', 'null'], + sample: '4111 1111 1111 1234', + token: 'CARD', + }, + { + id: 'card_security', + tier: 'restricted', + format: 'redact', + formats: ['redact', 'null'], + sample: '123', + token: 'CVV', + }, + { + id: 'bank_account', + tier: 'restricted', + format: 'last4', + formats: ['last4', 'redact', 'null'], + sample: 'NL91 ABNA 0417 1643 00', + token: 'BANK', + }, + { + id: 'credential', + tier: 'restricted', + format: 'redact', + formats: ['redact', 'null'], + sample: 'sk_live_4eC39HqLyjWDarjtT1', + token: 'CRED', + }, + { + id: 'special_category', + tier: 'restricted', + format: 'redact', + formats: ['redact', 'null'], + sample: 'Type 2 diabetes', + token: 'HEALTH', + }, + // identity + { + id: 'email', + tier: 'identity', + format: 'token', + formats: ['token', 'initial', 'redact', 'null'], + sample: 'jane.doe@example.com', + token: 'EMAIL', + }, + { + id: 'phone', + tier: 'identity', + format: 'token', + formats: ['token', 'last4', 'redact', 'null'], + sample: '+1 415-555-0132', + token: 'PHONE', + }, + { + id: 'person_name', + tier: 'identity', + format: 'token', + formats: ['token', 'initial', 'redact', 'null'], + sample: 'Jane Doe', + token: 'NAME', + }, + { + id: 'street_address', + tier: 'identity', + format: 'redact', + formats: ['redact', 'token', 'null'], + sample: '42 Market Street', + token: 'ADDR', + }, + { + id: 'date_of_birth', + tier: 'identity', + format: 'year', + formats: ['year', 'redact', 'token', 'null'], + sample: '1984-03-12', + token: 'DOB', + }, + { + id: 'ip_address', + tier: 'identity', + format: 'token', + formats: ['token', 'redact', 'null'], + sample: '203.0.113.42', + token: 'IP', + }, + { + id: 'username', + tier: 'identity', + format: 'token', + formats: ['token', 'initial', 'redact', 'null'], + sample: 'jdoe84', + token: 'USER', + }, + { + id: 'geo_precise', + tier: 'identity', + format: 'redact', + formats: ['redact', 'token', 'null'], + sample: '37.77493, -122.41942', + token: 'GEO', + }, + // quasi + { + id: 'postal_code', + tier: 'quasi', + format: 'redact', + formats: ['redact', 'token', 'null'], + sample: '94103', + token: 'ZIP', + }, + { + id: 'gender', + tier: 'quasi', + format: 'redact', + formats: ['redact', 'token', 'null'], + sample: 'Female', + token: 'GENDER', + }, + { + id: 'age', + tier: 'quasi', + format: 'redact', + formats: ['redact', 'null'], + sample: '41', + token: 'AGE', + }, + { + id: 'card_expiry', + tier: 'quasi', + format: 'redact', + formats: ['redact', 'year', 'null'], + sample: '07/28', + token: 'EXP', + }, + { + id: 'financial_personal', + tier: 'quasi', + format: 'redact', + formats: ['redact', 'null'], + sample: '$84,500', + token: 'FIN', + }, + // free text + { + id: 'free_text_pii', + tier: 'free_text', + format: 'span', + formats: ['span', 'redact', 'null'], + sample: 'Call Jane at 415-555-0132 or jane@example.com', + token: 'TEXT', + }, +]; + +const BY_ID = new Map(CLASSES.map(c => [c.id, c])); + +export function classDef(id: string | null | undefined): ClassDef | undefined { + return id ? BY_ID.get(id) : undefined; +} + +export function tierOf(id: string | null | undefined): Tier | null { + return classDef(id)?.tier ?? null; +} + +export function classesByTier(): { tier: Tier; classes: ClassDef[] }[] { + return TIERS.map(tier => ({ + tier, + classes: CLASSES.filter(c => c.tier === tier), + })); +} + +/** A Compliance baseline as Data Masking names it (first run, rules). */ +export interface BaselineDef { + id: string; + /** The framework, as admins say it. */ + name: string; + icon: string; + /** + * Longest admin grant the baseline allows, in hours. Mirrors the + * baseline definitions' apply.grant_max_hours in df-masking. + */ + grantMaxHours: number; +} + +export const BASELINES: BaselineDef[] = [ + { id: 'hipaa', name: 'HIPAA', icon: 'health_and_safety', grantMaxHours: 8 }, + { id: 'pci_dss', name: 'PCI DSS', icon: 'credit_card', grantMaxHours: 8 }, + { id: 'glba', name: 'GLBA', icon: 'account_balance', grantMaxHours: 24 }, + { id: 'cjis', name: 'CJIS', icon: 'gavel', grantMaxHours: 8 }, + { id: 'gdpr', name: 'GDPR', icon: 'privacy_tip', grantMaxHours: 24 }, + { id: 'soc2', name: 'SOC 2', icon: 'verified_user', grantMaxHours: 24 }, + { + id: 'iso27001', + name: 'ISO 27001', + icon: 'workspace_premium', + grantMaxHours: 24, + }, +]; + +export function baselineDef(id: string): BaselineDef | undefined { + return BASELINES.find(b => b.id === id); +} diff --git a/src/app/adf-data-masking/data-masking.forbidden.spec.ts b/src/app/adf-data-masking/data-masking.forbidden.spec.ts new file mode 100644 index 000000000..5d6c240f8 --- /dev/null +++ b/src/app/adf-data-masking/data-masking.forbidden.spec.ts @@ -0,0 +1,251 @@ +/** + * A 403 on any Data Masking write (a restricted admin, an admin who is not + * the super admin, an AI session) runs through the real HTTP pipeline (case + * and error interceptors, the API service) and must show the server's own + * message in one error toast, never sign the admin out, and leave the screen + * as the server has it. + */ +import { TestBed } from '@angular/core/testing'; +import { + HttpClient, + provideHttpClient, + withInterceptors, +} from '@angular/common/http'; +import { + HttpTestingController, + TestRequest, + provideHttpClientTesting, +} from '@angular/common/http/testing'; +import { Router, provideRouter } from '@angular/router'; +import { caseInterceptor } from 'src/app/shared/interceptors/case.interceptor'; +import { errorInterceptor } from 'src/app/shared/interceptors/error.interceptor'; +import { DfUserDataService } from 'src/app/shared/services/df-user-data.service'; +import { DfTrialService } from 'src/app/shared/services/df-trial.service'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { DfDataMaskingApiService } from './data-masking-api.service'; +import { DataMaskingStore, summarize } from './data-masking.store'; +import { DmUndoToastService } from './df-dm-undo-toast/df-dm-undo-toast.component'; +import { column, dmTransloco } from './dm-testing'; +import { + AccessSettings, + DetectionRule, + MaskedColumn, + MaskingOverview, +} from './data-masking.types'; + +const MESSAGE = 'Only a system administrator can change Data Masking settings.'; + +const overview = (over: Partial = {}): MaskingOverview => ({ + mode: 'on', + scan: { status: 'idle', columns: 2, finishedAt: '2026-10-28T00:00:00Z' }, + counts: { masked: 1, review: 1, clear: 0 }, + you: { canGrant: true, seesRealValues: false }, + ...over, +}); + +describe('DataMaskingStore: a 403 on any write', () => { + let store: DataMaskingStore; + let http: HttpTestingController; + let snack: { openSnackBar: jest.Mock; openError: jest.Mock }; + let user: { clearToken: jest.Mock; userData: null }; + let navigate: jest.SpyInstance; + + beforeEach(() => { + snack = { openSnackBar: jest.fn(), openError: jest.fn() }; + user = { clearToken: jest.fn(), userData: null }; + TestBed.configureTestingModule({ + imports: [dmTransloco()], + providers: [ + provideRouter([]), + provideHttpClient( + withInterceptors([caseInterceptor, errorInterceptor]) + ), + provideHttpClientTesting(), + DataMaskingStore, + DfDataMaskingApiService, + { provide: DfSnackbarService, useValue: snack }, + { provide: DmUndoToastService, useValue: { open: jest.fn() } }, + { provide: DfUserDataService, useValue: user }, + { provide: DfTrialService, useValue: { markLocked: jest.fn() } }, + ], + }); + TestBed.inject(HttpClient); + http = TestBed.inject(HttpTestingController); + navigate = jest.spyOn(TestBed.inject(Router), 'navigate'); + store = TestBed.inject(DataMaskingStore); + store.overview = overview(); + }); + + /** DreamFactory's error envelope for a refused write. */ + const forbid = (req: TestRequest, message = MESSAGE) => + req.flush( + { error: { code: 403, message, status_code: 403 } }, + { status: 403, statusText: 'Forbidden' } + ); + + const expectServerToast = (message = MESSAGE) => { + expect(snack.openError).toHaveBeenCalledTimes(1); + expect(snack.openError.mock.calls[0][0].message).toBe(message); + expect(snack.openError.mock.calls[0][0].status).toBe(403); + // Forbidden is not signed out: no token cleared, no trip to login. + expect(user.clearToken).not.toHaveBeenCalled(); + expect(navigate).not.toHaveBeenCalled(); + }; + + const seed = (): MaskedColumn[] => { + const list = [ + column({ + id: 'crm.a.dob', + service: 'crm', + table: 'a', + column: 'dob', + verdict: 'review', + class: 'date_of_birth', + tier: 'identity', + }), + column({ id: 'crm.a.ssn', service: 'crm', table: 'a', column: 'ssn' }), + ]; + store.columns.set('crm', list); + store.services = [ + { service: 'crm', label: 'CRM', tables: [summarize('a', list)] }, + ]; + return list; + }; + + afterEach(() => http.verify({ ignoreCancelled: true })); + + it('mode: back to the mode it had', () => { + store.setMode('preview'); + expect(store.overview?.mode).toBe('preview'); + forbid( + http.expectOne(r => r.method === 'PATCH'), + 'Only the super admin can lower the masking mode.' + ); + expect(store.overview?.mode).toBe('on'); + expect(store.modeSaving).toBe(false); + expectServerToast('Only the super admin can lower the masking mode.'); + }); + + it('scan: no scan shown as running', () => { + store.scan(); + forbid(http.expectOne(r => r.url.endsWith('/data_masking/scan'))); + expect(store.scanning).toBe(false); + expectServerToast(); + }); + + it('column decision: the column, the rail and the counts as they were', () => { + const [dob] = seed(); + store.decide(dob, null); + expect(store.counts.review).toBe(0); + forbid( + http.expectOne(r => r.url.endsWith('/data_masking/column')), + 'Only a system administrator can mark a column not sensitive.' + ); + const now = store.columns.get('crm')![0]; + expect(now.verdict).toBe('review'); + expect(now.class).toBe('date_of_birth'); + expect(store.counts.review).toBe(1); + expect(store.services[0].tables[0].columns.review).toBe(1); + expectServerToast( + 'Only a system administrator can mark a column not sensitive.' + ); + }); + + it('access: the confirmed settings come back and the view drops its choice', () => { + const confirmed: AccessSettings = { + admins: { mode: 'none', grants: [], available: [] }, + ai: 'masked', + roles: [], + version: 'v1', + }; + store.loadAccess(); + http.expectOne(r => r.method === 'GET').flush(confirmed); + const onError = jest.fn(); + store.updateAccess( + { ai: 'same_as_admins' }, + { ...confirmed, ai: 'same_as_admins' }, + onError + ); + forbid( + http.expectOne(r => r.method === 'PATCH'), + 'Only the super admin can give access.' + ); + expect(store.access?.ai).toBe('masked'); + expect(onError).toHaveBeenCalled(); + expectServerToast('Only the super admin can give access.'); + }); + + it('mask format: the formats as they were', () => { + store.loadMasks(); + http + .expectOne(r => r.method === 'GET') + .flush({ + resource: [ + { + class: 'email', + tier: 'identity', + format: 'initial', + formats_allowed: ['initial', 'redact'], + }, + ], + }); + store.setMask('email', 'redact'); + expect(store.masks![0].format).toBe('redact'); + forbid(http.expectOne(r => r.method === 'PATCH')); + expect(store.masks![0].format).toBe('initial'); + expectServerToast(); + }); + + it('add rule: the toast carries the message; the rule list is unchanged', () => { + store.rules = []; + let failed = false; + store + .addRule({ match: 'name', pattern: '^emp_ssn$', class: 'national_id' }) + .subscribe({ error: () => (failed = true) }); + forbid(http.expectOne(r => r.url.endsWith('/data_masking/rule'))); + expect(failed).toBe(true); + expect(store.rules).toEqual([]); + expectServerToast(); + }); + + it('delete rule: the rule comes back', () => { + const rule: DetectionRule = { + id: 7, + match: 'name', + pattern: '^emp_ssn$', + class: 'national_id', + matches: 0, + }; + store.rules = [rule]; + store.deleteRule(rule); + expect(store.rules).toEqual([]); + forbid(http.expectOne(r => r.method === 'DELETE')); + expect(store.rules).toEqual([rule]); + expectServerToast(); + }); + + it('delete and unmask: when unmasking is refused, the server says why', () => { + const [, ssn] = seed(); + const rule: DetectionRule = { + id: 7, + match: 'name', + pattern: '^ssn$', + class: 'national_id', + matches: 1, + }; + store.rules = [rule]; + store.deleteRule(rule, [ssn]); + http.expectOne(r => r.method === 'DELETE').flush({ id: 7 }); + forbid( + http.expectOne(r => r.url.endsWith('/data_masking/column')), + 'Only a system administrator can mark a column not sensitive.' + ); + expectServerToast( + 'Only a system administrator can mark a column not sensitive.' + ); + // Not the generic "Unmasked 0 of 1 columns". + expect(snack.openSnackBar).not.toHaveBeenCalled(); + // The page re-reads what the server now has. + expect(http.match(r => r.method === 'GET').length).toBeGreaterThan(0); + }); +}); diff --git a/src/app/adf-data-masking/data-masking.resolver.ts b/src/app/adf-data-masking/data-masking.resolver.ts new file mode 100644 index 000000000..51b9a1f9e --- /dev/null +++ b/src/app/adf-data-masking/data-masking.resolver.ts @@ -0,0 +1,14 @@ +import { inject } from '@angular/core'; +import { ResolveFn } from '@angular/router'; +import { map } from 'rxjs'; +import { DfPaywallService } from 'src/app/shared/services/df-paywall.service'; + +/** + * 'paywall' when this instance has no data_masking system resource (the + * df-masking package is not there), on any license, like Rate Limiting, + * Scheduler and Reporting; otherwise 'ok'. + */ +export const dataMaskingResolver: ResolveFn<'paywall' | 'ok'> = () => + inject(DfPaywallService) + .activatePaywall('data_masking') + .pipe(map(missing => (missing ? 'paywall' : 'ok'))); diff --git a/src/app/adf-data-masking/data-masking.store.spec.ts b/src/app/adf-data-masking/data-masking.store.spec.ts new file mode 100644 index 000000000..b7a8205e8 --- /dev/null +++ b/src/app/adf-data-masking/data-masking.store.spec.ts @@ -0,0 +1,1077 @@ +import { + TestBed, + fakeAsync, + flush, + tick, + discardPeriodicTasks, +} from '@angular/core/testing'; +import { Subject, of, throwError } from 'rxjs'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { + DataMaskingStore, + ruleFootprint, + summarize, +} from './data-masking.store'; +import { DmUndoToastService } from './df-dm-undo-toast/df-dm-undo-toast.component'; +import { DfDataMaskingApiService } from './data-masking-api.service'; +import { column, dmTransloco } from './dm-testing'; +import { + AccessSettings, + ActivityEvent, + ActivityQuery, + DetectionRule, + MaskedColumn, + MaskingOverview, +} from './data-masking.types'; + +const overview = (over: Partial = {}): MaskingOverview => ({ + mode: 'preview', + scan: { status: 'idle', columns: 3, finishedAt: '2026-10-28T00:00:00Z' }, + counts: { masked: 1, review: 1, clear: 1 }, + you: { canGrant: true, seesRealValues: false }, + ...over, +}); + +const access = (ai: AccessSettings['ai']): AccessSettings => ({ + admins: { mode: 'none', grants: [], available: [] }, + ai, + roles: [], +}); + +const boom = () => throwError(() => ({ status: 500, message: 'boom' })); + +describe('DataMaskingStore', () => { + let store: DataMaskingStore; + let api: Record; + let snack: { openSnackBar: jest.Mock; openError: jest.Mock }; + let undo: { open: jest.Mock }; + + const seedColumns = () => { + const list: MaskedColumn[] = [ + column({ + id: 'crm.a.dob', + table: 'a', + column: 'dob', + verdict: 'review', + class: 'date_of_birth', + tier: 'identity', + }), + column({ id: 'crm.a.ssn', table: 'a', column: 'ssn' }), + ]; + store.columns.set('crm', list); + store.services = [ + { service: 'crm', label: 'CRM', tables: [summarize('a', list)] }, + ]; + store.overview = overview(); + return list; + }; + + beforeEach(() => { + api = { + overview: jest.fn(() => of(overview())), + tables: jest.fn(() => of([])), + columns: jest.fn(() => of([])), + setMode: jest.fn(), + scan: jest.fn(), + decide: jest.fn(), + access: jest.fn(), + updateAccess: jest.fn(), + masks: jest.fn(), + setMask: jest.fn(), + rules: jest.fn(), + builtInRules: jest.fn(() => + of({ ruleset: 'x', version: '1', classes: [] }) + ), + addRule: jest.fn(), + deleteRule: jest.fn(), + activity: jest.fn(() => of({ events: [], hasMore: false })), + tableCount: jest.fn(() => of(0)), + baselines: jest.fn(() => of([])), + }; + snack = { openSnackBar: jest.fn(), openError: jest.fn() }; + undo = { open: jest.fn() }; + TestBed.configureTestingModule({ + imports: [dmTransloco()], + providers: [ + DataMaskingStore, + { provide: DfDataMaskingApiService, useValue: api }, + { provide: DfSnackbarService, useValue: snack }, + { provide: DmUndoToastService, useValue: undo }, + ], + }); + store = TestBed.inject(DataMaskingStore); + }); + + describe('activity', () => { + it('reloads in place after a change while it is loaded', () => { + store.overview = overview(); + store.loadActivity(); + expect(api['activity']).toHaveBeenCalledTimes(1); + api['setMode'].mockReturnValue(of(overview({ mode: 'on' }))); + store.setMode('on'); + expect(api['activity']).toHaveBeenCalledTimes(2); + // Never back to null: the tab keeps showing the list. + expect(store.activity).not.toBeNull(); + }); + + it('stays unloaded when its tab was never opened', () => { + store.overview = overview(); + api['setMode'].mockReturnValue(of(overview({ mode: 'on' }))); + store.setMode('on'); + expect(api['activity']).not.toHaveBeenCalled(); + expect(store.activity).toBeNull(); + }); + }); + + describe('load failures', () => { + it.each([ + ['access', 'loadAccess', 'access'], + ['masks', 'loadMasks', 'masks'], + ['rules', 'loadRules', 'rules'], + ['activity', 'loadActivity', 'activity'], + ] as const)( + 'marks %s failed so its tab can offer retry', + (slice, loader, call) => { + api[call].mockReturnValue(boom()); + (store[loader] as () => void)(); + expect(store.failed.has(slice)).toBe(true); + api[call].mockReturnValue( + of( + slice === 'access' + ? access('masked') + : slice === 'activity' + ? { events: [], hasMore: false } + : [] + ) + ); + (store[loader] as () => void)(); + expect(store.failed.has(slice)).toBe(false); + } + ); + }); + + describe('write ordering', () => { + it('a late failure never rolls back a newer successful decision', () => { + const list = seedColumns(); + const first = new Subject(); + const second = new Subject(); + api['decide'].mockReturnValueOnce(first).mockReturnValueOnce(second); + + store.decide(list[0], 'date_of_birth'); // keep masked + store.decide(store.columns.get('crm')![0], 'email'); // then re-class + second.next({ + ...list[0], + verdict: 'masked', + class: 'email', + source: 'admin', + }); + second.complete(); + first.error({ status: 500 }); + + const dob = store.columns.get('crm')![0]; + expect(dob.verdict).toBe('masked'); + expect(dob.class).toBe('email'); + expect(store.counts.review).toBe(0); + expect(snack.openError).not.toHaveBeenCalled(); + }); + + it('if the newer decision fails, it rolls back to the last confirmed one', () => { + const list = seedColumns(); + const first = new Subject(); + const second = new Subject(); + api['decide'].mockReturnValueOnce(first).mockReturnValueOnce(second); + + store.decide(list[0], 'date_of_birth'); + store.decide(store.columns.get('crm')![0], 'email'); + first.next({ + ...list[0], + verdict: 'masked', + class: 'date_of_birth', + source: 'admin', + }); + first.complete(); + second.error({ status: 500 }); + + const dob = store.columns.get('crm')![0]; + expect(dob.class).toBe('date_of_birth'); + expect(dob.verdict).toBe('masked'); + expect(store.counts.review).toBe(0); + expect(snack.openError).toHaveBeenCalledTimes(1); + }); + + it('access: writes go one at a time, each with the latest version', () => { + store.overview = overview(); + api['access'].mockReturnValue(of({ ...access('masked'), version: 'v1' })); + store.loadAccess(); + const first = new Subject(); + const second = new Subject(); + api['updateAccess'] + .mockReturnValueOnce(first) + .mockReturnValueOnce(second); + + store.updateAccess({ ai: 'same_as_admins' }, access('same_as_admins')); + store.updateAccess({ ai: 'masked' }, access('masked')); + // The second waits for the first: it must carry the version the + // first produces, or the server would refuse it as stale. + expect(api['updateAccess']).toHaveBeenCalledTimes(1); + expect(api['updateAccess'].mock.calls[0][0]).toEqual({ + ai: 'same_as_admins', + version: 'v1', + }); + // The newer optimistic state stays on screen meanwhile. + expect(store.access?.ai).toBe('masked'); + first.next({ ...access('same_as_admins'), version: 'v2' }); + first.complete(); + expect(store.access?.ai).toBe('masked'); + expect(api['updateAccess'].mock.calls[1][0]).toEqual({ + ai: 'masked', + version: 'v2', + }); + second.next({ ...access('masked'), version: 'v3' }); + second.complete(); + expect(store.access?.ai).toBe('masked'); + // One toast for the burst. + expect(snack.openSnackBar).toHaveBeenCalledTimes(1); + }); + + it('access: a stale version (409) restores, drops queued writes and re-reads', () => { + store.overview = overview(); + api['access'].mockReturnValue(of({ ...access('masked'), version: 'v1' })); + store.loadAccess(); + const first = new Subject(); + api['updateAccess'].mockReturnValueOnce(first); + const onError1 = jest.fn(); + const onError2 = jest.fn(); + store.updateAccess( + { ai: 'same_as_admins' }, + access('same_as_admins'), + onError1 + ); + store.updateAccess({ ai: 'masked' }, access('masked'), onError2); + // Someone else changed access since this page read it. + api['access'].mockReturnValue( + of({ ...access('same_as_admins'), version: 'v9' }) + ); + first.error({ + status: 409, + message: + 'Access settings changed since you loaded them. Reload and try again.', + }); + expect(onError1).toHaveBeenCalled(); + expect(onError2).toHaveBeenCalled(); + expect(api['updateAccess']).toHaveBeenCalledTimes(1); + // The page already re-reads: a UI string says so, not the server's + // "Reload and try again". + expect(snack.openError).not.toHaveBeenCalled(); + expect(snack.openSnackBar).toHaveBeenCalledTimes(1); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'Access settings changed elsewhere. Showing the latest.', + 'warning' + ); + // Re-read: the screen shows what the server now has. + expect(api['access']).toHaveBeenCalledTimes(2); + expect(store.access?.ai).toBe('same_as_admins'); + expect(store.access?.version).toBe('v9'); + }); + + it('access: a failed save restores the confirmed settings and calls back', () => { + store.overview = overview(); + api['access'].mockReturnValue(of(access('masked'))); + store.loadAccess(); + api['updateAccess'].mockReturnValue(boom()); + const onError = jest.fn(); + store.updateAccess( + { ai: 'same_as_admins' }, + access('same_as_admins'), + onError + ); + expect(store.access?.ai).toBe('masked'); + expect(onError).toHaveBeenCalled(); + }); + }); + + describe('scan polling', () => { + it('never stacks requests and finishes exactly once', fakeAsync(() => { + store.overview = overview(); + const polls: Subject[] = []; + api['scan'].mockReturnValue( + of(overview({ scan: { status: 'running', progress: 0, columns: 3 } })) + ); + api['overview'].mockImplementation(() => { + const s = new Subject(); + polls.push(s); + return s; + }); + store.scan(); + tick(800); + tick(800); + tick(800); + // The first request is still pending, so no others were sent. + expect(polls.length).toBe(1); + polls[0].next( + overview({ scan: { status: 'running', progress: 50, columns: 3 } }) + ); + polls[0].complete(); + tick(800); + expect(polls.length).toBe(2); + polls[1].next( + overview({ + scan: { + status: 'idle', + columns: 3, + finishedAt: '2026-10-29T00:00:00Z', + lastStatus: 'done', + }, + }) + ); + polls[1].complete(); + tick(800); + tick(800); + expect(polls.length).toBe(2); + expect(api['tables']).toHaveBeenCalledTimes(1); + expect(snack.openSnackBar).toHaveBeenCalledTimes(1); + expect(store.scanning).toBe(false); + discardPeriodicTasks(); + })); + + it('polling that keeps failing gives up after its retries, once', fakeAsync(() => { + store.overview = overview(); + api['scan'].mockReturnValue( + of(overview({ scan: { status: 'running', progress: 10, columns: 3 } })) + ); + api['overview'].mockReturnValue(boom()); + store.scan(); + tick(800); + // Retrying: still shown as running. + expect(store.scanning).toBe(true); + tick(1000 + 2000 + 3000 + 4000); + expect(store.scanning).toBe(false); + expect(api['overview']).toHaveBeenCalledTimes(5); + expect(snack.openError).toHaveBeenCalledTimes(1); + tick(5000); + expect(api['overview']).toHaveBeenCalledTimes(5); + })); + + it('one failed poll does not abandon a running scan', fakeAsync(() => { + store.overview = overview({ + scan: { status: 'running', progress: 10, columns: 3 }, + }); + let n = 0; + api['overview'].mockImplementation(() => { + n++; + // A single 502 from the proxy, then the scan keeps running. + return n === 1 + ? throwError(() => ({ status: 502 })) + : of( + overview({ + scan: { status: 'running', progress: 50, columns: 3 }, + }) + ); + }); + (store as unknown as { startPolling(own: boolean): void }).startPolling( + true + ); + tick(800 * 4); + expect(n).toBeGreaterThan(1); + expect(store.scanning).toBe(true); + expect(snack.openError).not.toHaveBeenCalled(); + discardPeriodicTasks(); + })); + + it('a poll issued before a mode change cannot put the old mode back', fakeAsync(() => { + store.overview = overview({ + mode: 'on', + scan: { status: 'running', progress: 40, columns: 3 }, + }); + const poll = new Subject(); + api['overview'].mockReturnValue(poll); + api['tables'].mockReturnValue(of([])); + (store as unknown as { startPolling(own: boolean): void }).startPolling( + true + ); + tick(800); // poll #1 is in flight (server still says On) + const patch = new Subject(); + api['setMode'].mockReturnValue(patch); + store.setMode('off'); + patch.next( + overview({ + mode: 'off', + scan: { status: 'running', progress: 60, columns: 3 }, + }) + ); + patch.complete(); + expect(store.mode).toBe('off'); + // The slower poll lands last: answered before the PATCH ran, and the + // scan's final poll at that. + poll.next(overview({ mode: 'on' })); + poll.complete(); + flush(); + expect(store.mode).toBe('off'); + discardPeriodicTasks(); + })); + }); + + describe('first scan', () => { + const unscanned = () => + overview({ + mode: 'off', + scan: { status: 'idle', progress: null, finishedAt: null, columns: 0 }, + counts: { masked: 0, review: 0, clear: 0 }, + }); + const scannedTables = () => [ + { + service: 'crm', + label: 'CRM', + tables: [summarize('a', [column({ table: 'a' })])], + }, + ]; + + it('covered services listed with no tables are not a scan', () => { + store.overview = unscanned(); + store.services = [{ service: 'crm', label: 'CRM', tables: [] }]; + expect(store.scanned).toBe(false); + expect(store.unscanned).toBe(true); + expect(store.firstRun).toBe(true); + expect(store.unscannedServices.map(s => s.service)).toEqual(['crm']); + }); + + it('a scanned database with no tables is not "not scanned yet"', () => { + store.overview = overview(); + store.services = [ + ...scannedTables(), + // The built-in SQLite `db` service of a default install: scanned, empty. + { + service: 'db', + label: 'Local SQL Database', + scannedAt: '2026-10-29T04:55:58Z', + tables: [], + }, + { service: 'hr', label: 'HR', scannedAt: null, tables: [] }, + ]; + expect(store.unscannedServices.map(s => s.service)).toEqual(['hr']); + }); + + it('a scan time alone proves a scan (every database empty)', () => { + store.overview = unscanned(); + store.services = [ + { + service: 'db', + label: 'DB', + scannedAt: '2026-10-29T04:55:58Z', + tables: [], + }, + ]; + expect(store.scanned).toBe(true); + expect(store.unscannedServices).toEqual([]); + }); + + it('a scan of empty databases found no columns (masking has nothing to act on)', () => { + store.overview = overview({ + mode: 'off', + scan: { + status: 'idle', + columns: 0, + finishedAt: '2026-10-29T04:55:58Z', + }, + counts: { masked: 0, review: 0, clear: 0 }, + }); + store.services = [ + { + service: 'db', + label: 'DB', + scannedAt: '2026-10-29T04:55:58Z', + tables: [], + }, + ]; + expect(store.scanned).toBe(true); + expect(store.foundColumns).toBe(false); + store.overview = overview(); + store.services = scannedTables(); + expect(store.foundColumns).toBe(true); + }); + + it('says Preview is on when the first scan moved the mode', fakeAsync(() => { + store.overview = unscanned(); + store.services = [{ service: 'crm', label: 'CRM', tables: [] }]; + api['scan'].mockReturnValue( + of({ + ...unscanned(), + scan: { status: 'running', progress: 0, columns: 0 }, + }) + ); + // The server switches Off -> Preview after the first scan ever. + api['overview'].mockReturnValue(of(overview({ mode: 'preview' }))); + api['tables'].mockReturnValue(of(scannedTables())); + api['columns'].mockReturnValue(of([column({ table: 'a' })])); + store.scan(); + expect(store.firstRun).toBe(true); + tick(800); + expect(store.mode).toBe('preview'); + expect(store.settling).toBe(false); + expect(store.firstRun).toBe(false); + expect(store.unscanned).toBe(false); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'Scan finished. Preview is on.', + 'success' + ); + discardPeriodicTasks(); + })); + + it('a later scan only says it finished', fakeAsync(() => { + store.overview = overview(); + store.services = scannedTables(); + api['scan'].mockReturnValue( + of(overview({ scan: { status: 'running', progress: 0, columns: 3 } })) + ); + api['overview'].mockReturnValue( + of( + overview({ + scan: { + status: 'idle', + columns: 3, + finishedAt: '2026-10-29T01:00:00Z', + lastStatus: 'done', + }, + }) + ) + ); + api['tables'].mockReturnValue(of(scannedTables())); + api['columns'].mockReturnValue(of([column({ table: 'a' })])); + store.scan(); + tick(800); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'Scan finished', + 'success' + ); + discardPeriodicTasks(); + })); + + it('scans one service when asked, and never twice at once', () => { + store.overview = overview(); + api['scan'].mockReturnValue(new Subject()); + store.scan('hr'); + store.scan('hr'); + expect(api['scan']).toHaveBeenCalledTimes(1); + expect(api['scan']).toHaveBeenCalledWith('hr'); + }); + + it('counts only databases with masked or review columns', () => { + store.services = [ + ...scannedTables(), + { + service: 'store', + label: 'Store', + tables: [ + summarize('t', [ + column({ verdict: 'clear', class: null, tier: null }), + ]), + ], + }, + { service: 'hr', label: 'HR', tables: [] }, + ]; + expect(store.maskedDatabases).toBe(1); + }); + + it('reads coverage from the overview', () => { + store.overview = overview({ + coverage: { + covered: [{ service: 'crm', label: 'CRM', type: 'pgsql' }], + uncovered: [{ service: 'm', label: 'Mongo', type: 'mongodb' }], + }, + }); + expect(store.uncovered.map(u => u.type)).toEqual(['mongodb']); + store.overview = overview(); + expect(store.uncovered).toEqual([]); + }); + }); + + describe('mode', () => { + it('holds further changes while one is saving', () => { + store.overview = overview(); + const pending = new Subject(); + api['setMode'].mockReturnValue(pending); + store.setMode('on'); + expect(store.modeSaving).toBe(true); + store.setMode('off'); + expect(api['setMode']).toHaveBeenCalledTimes(1); + pending.next(overview({ mode: 'on' })); + pending.complete(); + expect(store.modeSaving).toBe(false); + expect(store.mode).toBe('on'); + }); + }); + + describe('scan outcome', () => { + const tables = () => [ + { + service: 'crm', + label: 'CRM', + scannedAt: '2026-10-28T00:00:00Z', + tables: [summarize('a', [column({ table: 'a' })])], + }, + ]; + beforeEach(() => { + store.overview = overview(); + store.services = tables(); + api['scan'].mockReturnValue( + of(overview({ scan: { status: 'running', progress: 0, columns: 3 } })) + ); + api['tables'].mockReturnValue(of(tables())); + api['columns'].mockReturnValue(of([column({ table: 'a' })])); + }); + const toasts = () => + snack.openSnackBar.mock.calls.map(c => [c[0], c[1]] as const); + + it('a scan that failed (no new finished_at) is not announced as finished', fakeAsync(() => { + // An older backend: no last_status, and finished_at still names the + // previous successful scan. + api['overview'].mockReturnValue(of(overview())); + store.scan('crm'); + tick(800); + flush(); + expect(toasts().map(t => t[0])).not.toContain('Scan finished'); + expect(toasts()).toEqual([["Scan didn't finish.", 'warning']]); + discardPeriodicTasks(); + })); + + it('names the database a scan could not read', fakeAsync(() => { + api['overview'].mockReturnValue( + of( + overview({ + scan: { + status: 'idle', + columns: 3, + finishedAt: '2026-10-28T00:00:00Z', + lastStatus: 'failed', + failed: [ + { service: 'orders_db', label: 'Orders', reason: 'refused' }, + ], + }, + }) + ) + ); + store.scan('orders_db'); + tick(800); + flush(); + expect(toasts()).toEqual([ + ["Scan failed. Couldn't read Orders.", 'warning'], + ]); + discardPeriodicTasks(); + })); + + it('a partial scan finishes with a warning naming what it could not read', fakeAsync(() => { + api['overview'].mockReturnValue( + of( + overview({ + scan: { + status: 'idle', + columns: 3, + finishedAt: '2026-10-29T00:00:00Z', + lastStatus: 'partial', + failed: [ + { service: 'hr', label: 'HR' }, + { service: 'erp', label: 'ERP' }, + ], + }, + }) + ) + ); + store.scan(); + tick(800); + flush(); + expect(toasts()).toEqual([ + ["Scan finished. Couldn't read HR and ERP.", 'warning'], + ]); + discardPeriodicTasks(); + })); + + it("lists a database the scan could not read under Couldn't scan, not Not scanned yet", () => { + store.services = [ + ...tables(), + { + service: 'hr', + label: 'HR', + scannedAt: null, + scanError: 'Login failed', + tables: [], + }, + { + service: 'new', + label: 'New', + scannedAt: null, + scanError: null, + tables: [], + }, + ]; + expect(store.failedServices.map(s => s.service)).toEqual(['hr']); + expect(store.unscannedServices.map(s => s.service)).toEqual(['new']); + expect(store.scanErrorOf(store.services[1])).toBe('Login failed'); + }); + + it("a later scan keeps the admin's place (no view reset)", fakeAsync(() => { + const resets = jest.fn(); + store.viewReset$.subscribe(resets); + api['overview'].mockReturnValue( + of( + overview({ + scan: { + status: 'idle', + columns: 3, + finishedAt: '2026-10-29T00:00:00Z', + lastStatus: 'done', + }, + }) + ) + ); + const before = store.structure; + store.scan(); + tick(800); + flush(); + expect(resets).not.toHaveBeenCalled(); + expect(store.structure).toBeGreaterThan(before); + discardPeriodicTasks(); + })); + }); + + describe('background scans (page left open)', () => { + beforeEach(() => { + store.loading = false; + store.overview = overview(); + store.services = []; + }); + + it('new results reload the lists in place, quietly', fakeAsync(() => { + const resets = jest.fn(); + store.viewReset$.subscribe(resets); + api['overview'].mockReturnValue( + of( + overview({ + scan: { + status: 'idle', + columns: 4, + finishedAt: '2026-10-29T02:00:00Z', + lastStatus: 'done', + }, + }) + ) + ); + api['tables'].mockReturnValue( + of([{ service: 'inv', label: 'Inventory', scannedAt: 'x', tables: [] }]) + ); + store.startWatching(); + tick(30_000); + expect(api['tables']).toHaveBeenCalledTimes(1); + expect(store.services.map(s => s.service)).toEqual(['inv']); + expect(resets).not.toHaveBeenCalled(); + expect(snack.openSnackBar).not.toHaveBeenCalled(); + // Nothing new: nothing reloaded. + tick(30_000); + expect(api['tables']).toHaveBeenCalledTimes(1); + store.stopWatching(); + discardPeriodicTasks(); + })); + + it('a scan running elsewhere is followed to its end', fakeAsync(() => { + let n = 0; + api['overview'].mockImplementation(() => { + n++; + return of( + n < 3 + ? overview({ + scan: { status: 'running', progress: 50, columns: 3 }, + }) + : overview({ + scan: { + status: 'idle', + columns: 3, + finishedAt: '2026-10-29T03:00:00Z', + lastStatus: 'done', + }, + }) + ); + }); + store.startWatching(); + tick(30_000); + expect(store.scanning).toBe(true); + tick(800); + tick(800); + expect(store.scanning).toBe(false); + expect(api['tables']).toHaveBeenCalledTimes(1); + store.stopWatching(); + discardPeriodicTasks(); + })); + }); + + describe('decisions', () => { + it('not sensitive offers Undo, which masks the column again with its class', () => { + const list = seedColumns(); + api['decide'].mockImplementation((id: string, cls: string | null) => + of({ + ...list.find(c => c.id === id)!, + class: cls, + verdict: cls ? 'masked' : 'clear', + source: 'admin', + }) + ); + store.decide(list[1], null); // ssn, masked as national_id + expect(undo.open).toHaveBeenCalledTimes(1); + expect(undo.open.mock.calls[0][0]).toBe('ssn marked not sensitive'); + expect(undo.open.mock.calls[0][1]).toBe('Undo'); + undo.open.mock.calls[0][2](); + expect(api['decide']).toHaveBeenLastCalledWith( + 'crm.a.ssn', + 'national_id' + ); + expect(store.columns.get('crm')![1].verdict).toBe('masked'); + // Masking (keep or re-class) needs no undo toast. + store.decide(store.columns.get('crm')![0], 'date_of_birth'); + expect(undo.open).toHaveBeenCalledTimes(1); + }); + + it('an older success never becomes the rollback point after a newer one settled', () => { + const list = seedColumns(); + const a = new Subject(); + const b = new Subject(); + const c = new Subject(); + api['decide'] + .mockReturnValueOnce(a) + .mockReturnValueOnce(b) + .mockReturnValueOnce(c); + const at = (cls: string) => + ({ + ...list[0], + verdict: 'masked', + class: cls, + source: 'admin', + }) as MaskedColumn; + store.decide(list[0], 'email'); + store.decide(store.columns.get('crm')![0], 'phone'); + b.next(at('phone')); + b.complete(); + a.next(at('email')); // late: older than what the server has now + a.complete(); + store.decide(store.columns.get('crm')![0], 'username'); + c.error({ status: 500 }); + expect(store.columns.get('crm')![0].class).toBe('phone'); + }); + }); + + describe('rules', () => { + const rule = (over: Partial = {}): DetectionRule => ({ + id: 1, + match: 'name', + pattern: '(^|_)id$', + class: 'username', + matches: 2, + ...over, + }); + const ev = (lines: [number, string, string][]) => ({ + total: lines.reduce((n, l) => n + l[0], 0), + thresholdMask: 70, + thresholdReview: 40, + lines: lines.map(([points, kind, text]) => ({ + points, + kind: kind as 'name', + text, + })), + }); + + it('a failed rule delete puts back only that rule', () => { + const a = rule({ id: 1, pattern: '^a$' }); + const b = rule({ id: 2, pattern: '^b$' }); + store.rules = [{ ...a }]; + const del = new Subject(); + api['deleteRule'].mockReturnValue(del); + api['addRule'].mockReturnValue(of({ ...b })); + store.deleteRule(store.rules[0]); + store + .addRule({ match: 'name', pattern: '^b$', class: 'email' }) + .subscribe(); + del.error({ status: 500 }); + expect(store.rules?.map(r => r.id)).toEqual([1, 2]); + }); + + it('knows which columns a rule masked, and which nothing else would flag', () => { + const r = rule(); + const cols = [ + // Masked by the rule alone: safe to unmask with it. + column({ + id: 's.t.id', + verdict: 'masked', + evidence: ev([[70, 'context', 'Custom rule (^|_)id$']]), + }), + // The rule plus a weak signal: would be "review" without it. + column({ + id: 's.t.user_id', + verdict: 'masked', + evidence: ev([ + [70, 'context', 'Custom rule (^|_)id$'], + [45, 'name', 'Column name'], + ]), + }), + // Masked anyway (strong name evidence): not the rule's doing. + column({ + id: 's.t.ssn_id', + verdict: 'masked', + evidence: ev([ + [70, 'context', 'Custom rule (^|_)id$'], + [80, 'name', 'Column name "ssn"'], + ]), + }), + // An admin decision is never touched. + column({ + id: 's.t.cid', + verdict: 'masked', + source: 'admin', + evidence: ev([[70, 'context', 'Custom rule (^|_)id$']]), + }), + ]; + const f = ruleFootprint(r, cols); + expect(f.masked.map(c => c.id)).toEqual(['s.t.id', 's.t.user_id']); + expect(f.ruleOnly.map(c => c.id)).toEqual(['s.t.id']); + // Value and baseline rules write their own evidence lines. + expect( + ruleFootprint(rule({ match: 'value', source: 'baseline:hipaa' }), [ + column({ + id: 's.t.x', + evidence: ev([ + [70, 'values', 'Baseline value rule (^|_)id$ (hipaa)'], + ]), + }), + ]).masked.length + ).toBe(1); + }); + + it('delete and unmask marks those columns not sensitive, then says so', () => { + store.overview = overview(); + const cols = [column({ id: 'crm.a.id', table: 'a', column: 'id' })]; + store.columns.set('crm', cols); + store.services = [ + { service: 'crm', label: 'CRM', tables: [summarize('a', cols)] }, + ]; + store.rules = [rule()]; + api['deleteRule'].mockReturnValue(of({})); + api['decide'].mockReturnValue(of({ ...cols[0], verdict: 'clear' })); + api['tables'].mockReturnValue(of(store.services)); + api['columns'].mockReturnValue(of(cols)); + store.deleteRule(store.rules[0], cols); + expect(api['decide']).toHaveBeenCalledWith('crm.a.id', null); + expect(snack.openSnackBar).toHaveBeenCalledWith( + 'Rule deleted. 1 column unmasked.', + 'success' + ); + }); + }); + + describe('activity paging', () => { + const ev = (id: number, kind: ActivityEvent['kind']): ActivityEvent => ({ + id, + at: new Date(Date.UTC(2026, 9, 29, 0, 0, id)).toISOString(), + actor: 'x', + kind, + text: kind, + }); + + it("asks the server for a filter's kinds and pages back with before", () => { + api['activity'].mockImplementation((q: ActivityQuery) => + of( + q.before + ? { events: [ev(1, 'blocked')], hasMore: false } + : { events: [ev(3, 'blocked'), ev(2, 'blocked')], hasMore: true } + ) + ); + store.showActivity(['blocked']); + expect(api['activity']).toHaveBeenLastCalledWith({ + kinds: ['blocked'], + limit: 200, + }); + expect(store.activity?.map(e => e.id)).toEqual([3, 2]); + expect(store.activityHasMore).toBe(true); + store.loadOlderActivity(); + expect(api['activity']).toHaveBeenLastCalledWith({ + kinds: ['blocked'], + limit: 200, + before: 2, + }); + expect(store.activity?.map(e => e.id)).toEqual([3, 2, 1]); + expect(store.activityHasMore).toBe(false); + }); + }); + + describe('activity refresh vs Load older', () => { + it('a refresh that lands after Load older keeps the older rows', () => { + const ev = (id: number): ActivityEvent => ({ + id, + at: new Date(Date.UTC(2026, 9, 29, 0, 0, id)).toISOString(), + actor: 'x', + kind: 'mode', + text: 'm', + }); + const first = new Subject<{ + events: ActivityEvent[]; + hasMore: boolean; + }>(); + api['activity'] + .mockReturnValueOnce(of({ events: [ev(4), ev(3)], hasMore: true })) + .mockReturnValueOnce(first) + .mockReturnValueOnce(of({ events: [ev(2), ev(1)], hasMore: false })); + store.showActivity(null); + store.invalidateActivity(); // a refresh goes out (slow) + store.loadOlderActivity(); // and the admin pages back meanwhile + expect(store.activity?.map(e => e.id)).toEqual([4, 3, 2, 1]); + first.next({ events: [ev(5), ev(4)], hasMore: true }); + first.complete(); + expect(store.activity?.map(e => e.id)).toEqual([5, 4, 3, 2, 1]); + expect(store.activityHasMore).toBe(false); + }); + }); + + describe('nothing to mask yet', () => { + it('before any scan, a default install whose only database is empty has nothing to scan', () => { + api['overview'].mockReturnValue( + of( + overview({ + mode: 'off', + scan: { status: 'idle', finishedAt: null, columns: 0 }, + counts: { masked: 0, review: 0, clear: 0 }, + coverage: { + covered: [ + { service: 'db', label: 'Local SQL Database', type: 'sqlite' }, + ], + uncovered: [], + }, + }) + ) + ); + api['tables'].mockReturnValue( + of([{ service: 'db', label: 'Local SQL Database', tables: [] }]) + ); + api['tableCount'].mockReturnValue(of(0)); + store.load(); + expect(api['tableCount']).toHaveBeenCalledWith('db'); + expect(store.nothingToScan).toBe(true); + // A database that could not answer is never assumed empty. + api['tableCount'].mockReturnValue(throwError(() => ({ status: 500 }))); + store.load(); + expect(store.nothingToScan).toBe(false); + }); + + it('after a scan, no tables anywhere means add a database', () => { + store.overview = overview(); + store.services = [ + { + service: 'db', + label: 'Local SQL Database', + scannedAt: 'x', + tables: [], + }, + ]; + expect(store.noTables).toBe(true); + store.services = [ + ...store.services, + { service: 'hr', label: 'HR', scannedAt: null, tables: [] }, + ]; + expect(store.noTables).toBe(false); + }); + }); +}); diff --git a/src/app/adf-data-masking/data-masking.store.ts b/src/app/adf-data-masking/data-masking.store.ts new file mode 100644 index 000000000..d519a5dbe --- /dev/null +++ b/src/app/adf-data-masking/data-masking.store.ts @@ -0,0 +1,1296 @@ +/** + * Shared state for the Data Masking page. The shell provides one instance; + * every tab injects it. Writes are optimistic: the UI moves first, the + * server confirms, and a failure rolls back with a single error toast. + */ +import { Injectable, OnDestroy, inject } from '@angular/core'; +import { TranslocoService } from '@ngneat/transloco'; +import { + EMPTY, + Observable, + Subject, + Subscription, + catchError, + defer, + exhaustMap, + filter, + finalize, + forkJoin, + from, + fromEvent, + interval, + map, + merge, + mergeMap, + of, + retry, + takeWhile, + tap, + timer, + toArray, +} from 'rxjs'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { isAppError, normalizeError } from 'src/app/shared/utilities/app-error'; +import { DfDataMaskingApiService } from './data-masking-api.service'; +import { MaskFormat, classDef } from './data-masking.catalog'; +import { + AccessPatch, + AccessSettings, + ActivityEvent, + ActivityKind, + BuiltInRules, + CoverageService, + DetectionRule, + MaskSetting, + MaskedColumn, + MaskingMode, + MaskingOverview, + ScanFailure, + ScanOutcome, + ServiceTables, + StripMark, + TableSummary, + Verdict, +} from './data-masking.types'; +import { DmUndoToastService } from './df-dm-undo-toast/df-dm-undo-toast.component'; + +export type ColumnFilter = 'review' | 'masked' | 'clear' | 'all'; + +/** Tab slices that load on their own and can fail on their own. */ +export type DmSlice = 'access' | 'masks' | 'rules' | 'activity'; + +/** A column-id filter set from elsewhere (a rule's Matches count). */ +export interface IdFilter { + label: string; + ids: Set; +} + +/** HTTP status of a failed request (AppError or raw), if any. */ +export function errorStatus(e: unknown): number | undefined { + return isAppError(e) ? e.status : (e as { status?: number })?.status; +} + +export const tableKey = (service: string, table: string) => + `${service}\u0000${table}`; + +/** Activity rows per page (the server's default). */ +export const ACTIVITY_PAGE = 200; +/** How often an open page checks for scans it did not start. */ +const WATCH_MS = 30_000; +/** Poll retries before a running scan is given up on. */ +const POLL_RETRIES = 4; + +export function stripMark(c: MaskedColumn): StripMark { + if (c.verdict === 'review') return 'review'; + if (c.verdict === 'clear' || !c.tier) return 'clear'; + return c.tier; +} + +export function summarize( + name: string, + cols: MaskedColumn[], + rows?: number | null +): TableSummary { + const columns = { masked: 0, review: 0, clear: 0 }; + for (const c of cols) columns[c.verdict]++; + return { + name, + rows, + columns, + strip: cols.map(c => ({ + column: c.column, + mark: stripMark(c), + tier: c.tier, + })), + }; +} + +/** + * The evidence line a rule writes on the columns it matches (the scanner's + * Rules::describe): "Custom rule P", "Custom value rule P", or + * "Baseline rule P (hipaa)". + */ +export function ruleEvidence(rule: DetectionRule): string { + const kind = rule.match === 'value' ? 'value rule' : 'rule'; + const source = rule.source ?? 'admin'; + return source.startsWith('baseline:') + ? `Baseline ${kind} ${rule.pattern} (${source.slice('baseline:'.length)})` + : `Custom ${kind} ${rule.pattern}`; +} + +/** What deleting a rule leaves behind. */ +export interface RuleFootprint { + /** Columns the rule masked (masked because of it, not decided by an admin). */ + masked: MaskedColumn[]; + /** Of those, the ones nothing else would flag: safe to unmask. */ + ruleOnly: MaskedColumn[]; +} + +export function ruleFootprint( + rule: DetectionRule, + columns: MaskedColumn[] +): RuleFootprint { + const text = ruleEvidence(rule); + const masked: MaskedColumn[] = []; + const ruleOnly: MaskedColumn[] = []; + for (const c of columns) { + if (c.source !== 'detected' || c.verdict !== 'masked') continue; + const lines = c.evidence?.lines ?? []; + const line = lines.find(l => l.points > 0 && l.text === text); + if (!line) continue; + const others = lines + .filter(l => l !== line) + .reduce((sum, l) => sum + l.points, 0); + if (others >= c.evidence.thresholdMask) continue; // masked anyway + masked.push(c); + if (others < c.evidence.thresholdReview) ruleOnly.push(c); + } + return { masked, ruleOnly }; +} + +interface AccessWrite { + patch: AccessPatch; + onError?: () => void; + toast: string; +} + +@Injectable() +export class DataMaskingStore implements OnDestroy { + private api = inject(DfDataMaskingApiService); + private snackbar = inject(DfSnackbarService); + private undoToast = inject(DmUndoToastService); + private transloco = inject(TranslocoService); + + overview: MaskingOverview | null = null; + services: ServiceTables[] = []; + /** Columns by service name. */ + columns = new Map(); + access: AccessSettings | null = null; + masks: MaskSetting[] | null = null; + rules: DetectionRule[] | null = null; + builtin: BuiltInRules | null = null; + /** Activity for the current filter, newest first; null until loaded. */ + activity: ActivityEvent[] | null = null; + /** Older activity exists for the current filter. */ + activityHasMore = false; + activityLoadingOlder = false; + + loading = true; + loadFailed = false; + /** The server's reason when the page could not load (e.g. not set up). */ + loadError: string | null = null; + /** Slices whose last load failed; their tab shows a retry state. */ + readonly failed = new Set(); + /** A mode PATCH is in flight; the toggle waits for it. */ + modeSaving = false; + + /** Columns-tab view state, kept here so other tabs can steer it. */ + filter: ColumnFilter | null = null; + idFilter: IdFilter | null = null; + /** Emits whenever a tab should re-snapshot its view (filter jump). */ + readonly viewReset$ = new Subject(); + /** Bumped on every column change so views can memoize cheaply. */ + version = 0; + /** + * Bumped when the service/table list is replaced (a scan finished in the + * background): views rebuild their lists but keep filter and selection. + */ + structure = 0; + + /** + * First run: tables each covered database lists, by service, once the + * probe answered (only asked before the first scan). A database that + * could not be asked is left out, so it never counts as empty. + */ + tableCounts: Map | null = null; + + private poll?: Subscription; + private watch?: Subscription; + /** + * Write sequence per target (column id, 'masks'). A response only lands + * if it belongs to the latest write for its target, so a slow failure + * can never roll back over a newer success. + */ + private seq = new Map(); + /** + * Bumped by every mode change. An overview requested before the latest + * change keeps the mode on screen, so a slow poll never puts an old mode + * back. + */ + private modeEpoch = 0; + + private nextSeq(key: string): number { + const n = (this.seq.get(key) ?? 0) + 1; + this.seq.set(key, n); + return n; + } + + private isLatest(key: string, n: number): boolean { + return this.seq.get(key) === n; + } + + t(key: string, params?: Record): string { + return this.transloco.translate(`dataMasking.${key}`, params); + } + + /** "1 column" / "3 columns" from plural..one|other. */ + plural(key: string, count: number): string { + return this.t(`plural.${key}.${count === 1 ? 'one' : 'other'}`, { count }); + } + + /** "hr", "hr and crm", "hr and 2 more". */ + nameList(names: string[]): string { + if (names.length <= 1) return names[0] ?? ''; + if (names.length === 2) { + return this.t('list.two', { a: names[0], b: names[1] }); + } + return this.t('list.more', { a: names[0], count: names.length - 1 }); + } + + get mode(): MaskingMode { + return this.overview?.mode ?? 'off'; + } + + get counts() { + return this.overview?.counts ?? { masked: 0, review: 0, clear: 0 }; + } + + /** + * Columns masked when masking is on: decided masked plus those still to + * review (they fail closed). The counts on screen stay disjoint; only the + * turn-on confirm uses this total. + */ + get maskedTotal(): number { + const c = this.counts; + return c.masked + c.review; + } + + /** Every column the last scan classified. */ + get totalColumns(): number { + const c = this.counts; + return c.masked + c.review + c.clear; + } + + /** + * Something has been classified. /table lists every covered service, a + * never-scanned one with `tables: []`, so a service row alone proves + * nothing: it takes a finished scan, a service with a scan time, or at + * least one classified table. + */ + get scanned(): boolean { + return ( + !!this.overview?.scan.finishedAt || + this.services.some(s => s.tables.length > 0 || !!s.scannedAt) + ); + } + + /** + * A scan has found at least one column. A finished scan of empty + * databases does not count: there is nothing to preview, and the server + * refuses On until a scan finds columns (decision 21c). + */ + get foundColumns(): boolean { + return this.totalColumns > 0 || (this.overview?.scan.columns ?? 0) > 0; + } + + /** Databases with at least one masked (or still-to-review) column. */ + get maskedDatabases(): number { + return this.services.filter(s => + s.tables.some(t => t.columns.masked + t.columns.review > 0) + ).length; + } + + /** Database services Data Masking cannot mask (non-SQL connectors). */ + get uncovered(): CoverageService[] { + return this.overview?.coverage?.uncovered ?? []; + } + + /** + * Covered services that have never been scanned (added since). A scanned + * database with no tables has a scan time and nothing to show; it is not + * "not scanned yet". One the last scan could not read is listed under + * "Couldn't scan" instead. + */ + get unscannedServices(): ServiceTables[] { + return this.services.filter( + s => !s.tables.length && !s.scannedAt && !this.scanErrorOf(s) + ); + } + + /** Covered services the last scan could not read. */ + get failedServices(): ServiceTables[] { + return this.services.filter(s => !!this.scanErrorOf(s)); + } + + /** Why a service could not be scanned: /table's scan_error, else the overview's list. */ + scanErrorOf(s: ServiceTables): string | null { + if (s.scanError) return s.scanError; + if (s.scanError === null) return null; + const f = this.overview?.scan.failed?.find(x => x.service === s.service); + return f ? f.reason || this.t('columns.scanFailed') : null; + } + + /** + * Scanned, and no covered database holds a table: there is nothing to + * mask until a database with tables is added. + */ + get noTables(): boolean { + if (!this.services.length) return false; + if (this.unscannedServices.length || this.failedServices.length) { + return false; + } + return this.services.every(s => !s.tables.length); + } + + /** + * Before the first scan: every covered database answered the table probe + * and none lists a table (a default install's built-in `db`), so a scan + * would find nothing. + */ + get nothingToScan(): boolean { + const covered = this.overview?.coverage?.covered ?? []; + const counts = this.tableCounts; + if (!covered.length || !counts) return false; + return covered.every(c => counts.get(c.service) === 0); + } + + /** Scan progress 0-100, or null while the server can't say yet. */ + get scanProgress(): number | null { + const p = this.overview?.scan.progress; + return typeof p === 'number' ? p : null; + } + + /** + * Nothing has been classified yet: masking has nothing to act on, so + * Preview and On wait for the first scan to finish. + */ + get unscanned(): boolean { + return !this.scanned && (this.overview?.scan.columns ?? 0) === 0; + } + + /** + * The server could not read its scan history: whether (and when) anything + * was scanned is unknown, so the screen must not look never-scanned or + * offer the first scan. + */ + get scanUnknown(): boolean { + return ( + !!this.overview?.scan.unknown || this.services.some(s => !!s.scanUnknown) + ); + } + + /** + * The first-run view: never scanned, or the first scan just finished and + * its results are still loading (so the page never flashes an empty + * filter between the last poll and the first table list). + */ + get firstRun(): boolean { + return !this.scanned || this.settling; + } + + /** The first scan finished and its tables/columns are loading. */ + settling = false; + + get scanning(): boolean { + return this.overview?.scan.status === 'running'; + } + + /** Can the signed-in admin lower the mode (On to Preview or Off)? */ + get canLower(): boolean { + return !!this.overview?.you.canGrant; + } + + load(): void { + this.loading = true; + this.loadError = null; + const epoch = this.modeEpoch; + forkJoin({ + overview: this.api.overview(), + services: this.api.tables(), + }).subscribe({ + next: ({ overview, services }) => { + this.applyOverview(overview, epoch); + this.setServices(services); + this.loadColumns().subscribe(() => { + this.loading = false; + this.loadFailed = false; + this.viewReset$.next(); + }); + if (overview.scan.status === 'running') this.startPolling(false); + if (this.unscanned) this.probeTables(); + }, + error: e => { + this.loading = false; + this.loadFailed = true; + const err = isAppError(e) ? e : normalizeError(e); + // "Not set up yet: run …" is worth showing in place; generic + // failures keep the plain message. + this.loadError = err.status === 503 ? err.message : null; + }, + }); + } + + private setServices(services: ServiceTables[]): void { + this.services = services; + this.structure++; + } + + private loadColumns(): Observable { + if (!this.services.length) return of(undefined); + return forkJoin( + this.services.map(s => + this.api.columns(s.service).pipe( + tap(cols => this.columns.set(s.service, cols)), + catchError(() => of([])) + ) + ) + ).pipe( + map(() => { + this.version++; + }) + ); + } + + /** + * First run only: ask each covered database how many tables it lists, so + * an instance whose only database is empty is sent to add one instead of + * scanning nothing. A database that cannot answer is assumed worth a scan. + */ + private probeTables(): void { + const covered = this.overview?.coverage?.covered ?? []; + // A real instance with many databases is never "nothing to scan". + if (!covered.length || covered.length > 5) return; + forkJoin( + covered.map(c => + this.api.tableCount(c.service).pipe( + map(n => [c.service, n] as const), + catchError(() => of(null)) + ) + ) + ).subscribe(rows => { + const counts = new Map(); + rows.forEach(r => r && counts.set(r[0], r[1])); + this.tableCounts = counts; + }); + } + + private tableIndex: { + version: number; + lists: MaskedColumn[][]; + map: Map; + } = { version: -1, lists: [], map: new Map() }; + + /** Every loaded column, in rail order. */ + allColumns(): MaskedColumn[] { + return this.services.flatMap(s => this.columns.get(s.service) ?? []); + } + + /** + * A table's columns, from an index rebuilt once per column change (or + * when a service's column list is replaced). + */ + tableColumns(service: string, table: string): MaskedColumn[] { + const lists = [...this.columns.values()]; + const idx = this.tableIndex; + const fresh = + idx.version === this.version && + idx.lists.length === lists.length && + idx.lists.every((l, i) => l === lists[i]); + if (!fresh) { + const map = new Map(); + for (const [svc, cols] of this.columns) { + for (const c of cols) { + const k = tableKey(svc, c.table); + const list = map.get(k); + if (list) list.push(c); + else map.set(k, [c]); + } + } + this.tableIndex = { version: this.version, lists, map }; + } + return this.tableIndex.map.get(tableKey(service, table)) ?? []; + } + + private summaryIndex: { + services: ServiceTables[] | null; + version: number; + map: Map; + } = { services: null, version: -1, map: new Map() }; + + /** A table's rail summary (counts and strip). */ + summaryOf(service: string, table: string): TableSummary | undefined { + const idx = this.summaryIndex; + if (idx.services !== this.services || idx.version !== this.version) { + const m = new Map(); + for (const s of this.services) { + for (const t of s.tables) m.set(tableKey(s.service, t.name), t); + } + this.summaryIndex = { + services: this.services, + version: this.version, + map: m, + }; + } + return this.summaryIndex.map.get(tableKey(service, table)); + } + + /** + * Take an overview response. One requested before the latest mode change + * (or while one is saving) keeps the mode on screen: it is older news. + */ + private applyOverview(o: MaskingOverview, epoch: number): void { + const stale = epoch !== this.modeEpoch || this.modeSaving; + this.overview = + stale && this.overview ? { ...o, mode: this.overview.mode } : o; + } + + // ---------------------------------------------------------------- mode + + setMode(mode: MaskingMode): void { + if (!this.overview || this.modeSaving || mode === this.overview.mode) { + return; + } + const prev = this.overview.mode; + this.modeEpoch++; + this.overview = { ...this.overview, mode }; + this.modeSaving = true; + this.api + .setMode(mode) + .pipe(finalize(() => (this.modeSaving = false))) + .subscribe({ + next: o => { + this.overview = { ...this.overview!, ...o }; + this.toast( + prev === 'on' && mode === 'preview' + ? 'toast.mode.paused' + : `toast.mode.${mode}` + ); + this.invalidateActivity(); + }, + error: e => { + this.overview = { ...this.overview!, mode: prev }; + this.fail(e); + }, + }); + } + + // ---------------------------------------------------------------- scan + + /** Mode before the running scan started: the first scan may move it. */ + private modeAtScan: MaskingMode | null = null; + /** finished_at before the running scan: tells an old backend's failure. */ + private finishedBefore: string | null = null; + + /** Scan every covered service, or just `service`. */ + scan(service?: string): void { + if (!this.overview || this.scanning) return; + this.modeAtScan = this.overview.mode; + this.finishedBefore = this.overview.scan.finishedAt ?? null; + this.overview = { + ...this.overview, + scan: { ...this.overview.scan, status: 'running', progress: 0 }, + }; + const epoch = this.modeEpoch; + this.api.scan(service).subscribe({ + next: o => { + this.applyOverview(o, epoch); + this.startPolling(true); + }, + error: e => { + this.overview = { + ...this.overview!, + scan: { ...this.overview!.scan, status: 'idle' }, + }; + this.fail(e); + }, + }); + } + + /** + * Poll the overview while a scan runs. exhaustMap never stacks requests + * behind a slow backend, a failed poll is retried with backoff (one proxy + * blip must not abandon the scan), and takeWhile(…, true) lets exactly + * one finished response through, so the finish work runs once. `own` + * scans (started here) announce how they ended; one found running in + * the background just refreshes the page. + */ + private startPolling(own: boolean): void { + this.poll?.unsubscribe(); + if (!own) { + this.modeAtScan = this.overview?.mode ?? null; + this.finishedBefore = this.overview?.scan.finishedAt ?? null; + } + let finished = false; + this.poll = timer(800, 800) + .pipe( + exhaustMap(() => { + const epoch = this.modeEpoch; + // defer: every retry is a fresh request. + return defer(() => this.api.overview(true)).pipe( + retry({ count: POLL_RETRIES, delay: (_e, i) => timer(1000 * i) }), + map(o => ({ o, epoch })) + ); + }), + takeWhile(({ o }) => o.scan.status === 'running', true) + ) + .subscribe({ + next: ({ o, epoch }) => { + this.applyOverview(o, epoch); + if (o.scan.status !== 'running') finished = true; + }, + complete: () => { + this.poll = undefined; + if (finished) this.finishScan(own); + }, + error: e => { + this.poll = undefined; + // Polling lost the scan after its retries: stop showing it as + // running so Scan now is usable again, and say so once. + if (this.overview) { + this.overview = { + ...this.overview, + scan: { ...this.overview.scan, status: 'idle' }, + }; + } + this.fail(e); + }, + }); + } + + /** How the scan that just finished went. */ + private outcome(): ScanOutcome { + const scan = this.overview?.scan; + if (scan?.lastStatus) return scan.lastStatus; + // A backend without last_status: a failed scan leaves finished_at alone. + return scan?.finishedAt && scan.finishedAt !== this.finishedBefore + ? 'done' + : 'failed'; + } + + private finishScan(own: boolean): void { + // The server switches Off to Preview after the first scan ever; the + // last poll already carried the new mode, so say why it moved. + const toPreview = + this.modeAtScan === 'off' && this.overview?.mode === 'preview'; + const outcome = this.outcome(); + const failed: ScanFailure[] = this.overview?.scan.failed ?? []; + this.modeAtScan = null; + const first = !this.services.some(s => s.tables.length || s.scannedAt); + this.settling = first && outcome !== 'failed'; + this.api.tables().subscribe({ + next: services => { + this.setServices(services); + this.loadColumns().subscribe(() => { + this.settling = false; + this.invalidateActivity(); + // The first results pick the opening filter; later ones keep the + // admin's place (filter, table) and only refresh the lists. + if (first) this.viewReset$.next(); + if (own || outcome !== 'done') + this.announce(outcome, failed, toPreview); + }); + }, + error: () => (this.settling = false), + }); + } + + private announce( + outcome: ScanOutcome, + failed: ScanFailure[], + toPreview: boolean + ): void { + const label = this.nameList(failed.map(f => f.label || f.service)); + if (outcome === 'done') { + this.toast(toPreview ? 'toast.scanDonePreview' : 'toast.scanDone'); + } else if (outcome === 'partial') { + this.warn('toast.scanPartial', { label }); + } else { + this.warn(label ? 'toast.scanFailed' : 'toast.scanFailedNoReason', { + label, + }); + } + } + + /** + * Keep an open page current: every 30 s while it is visible, and when the + * window regains focus, re-read the overview. A scan started elsewhere + * (a new database, the CLI) is followed to its end; results that changed + * since (finished_at or the covered databases) reload the lists in place. + */ + startWatching(): void { + if (this.watch || typeof window === 'undefined') return; + const visible = () => + typeof document === 'undefined' || document.visibilityState !== 'hidden'; + this.watch = merge( + interval(WATCH_MS), + fromEvent(window, 'focus'), + fromEvent(document, 'visibilitychange') + ) + .pipe( + filter( + () => + visible() && + !this.loading && + !!this.overview && + !this.poll && + !this.scanning + ), + exhaustMap(() => { + const epoch = this.modeEpoch; + return this.api.overview(true).pipe( + map(o => ({ o, epoch })), + catchError(() => EMPTY) + ); + }) + ) + .subscribe(({ o, epoch }) => this.onWatched(o, epoch)); + } + + stopWatching(): void { + this.watch?.unsubscribe(); + this.watch = undefined; + } + + private onWatched(o: MaskingOverview, epoch: number): void { + const prev = this.overview; + if (!prev || this.poll) return; + const covered = (x: MaskingOverview | null) => + (x?.coverage?.covered ?? []).map(c => c.service).join('\u0000'); + const changed = + (o.scan.finishedAt ?? null) !== (prev.scan.finishedAt ?? null) || + covered(o) !== covered(prev); + // Counts move with optimistic decisions; keep ours while any is saving. + const keepCounts = this.confirmed.size > 0; + this.applyOverview(keepCounts ? { ...o, counts: prev.counts } : o, epoch); + if (o.scan.status === 'running') { + this.startPolling(false); + } else if (changed) { + this.refreshTables(); + } + } + + /** Reload the table list and columns in place (no filter reset). */ + private refreshTables(): void { + this.api.tables().subscribe({ + next: services => { + this.setServices(services); + this.loadColumns().subscribe(() => this.invalidateActivity()); + }, + error: () => undefined, + }); + } + + // ------------------------------------------------------------ decisions + + /** + * Record an admin decision: a class id masks the column with that class, + * null marks it not sensitive. + */ + decide(col: MaskedColumn, cls: string | null): void { + const { service, id } = col; + // Roll back to what the server last confirmed, not to whatever an + // earlier optimistic write left on screen. + const before = this.confirmed.get(id) ?? this.findColumn(service, id); + if (!before) return; + const shown = this.findColumn(service, id) ?? before; + if (!this.confirmed.has(id)) { + // Every write so far is older than this baseline. + this.confirmed.set(id, { ...before }); + this.confirmedSeq.set(id, this.seq.get(id) ?? 0); + } + const n = this.nextSeq(id); + const verdict: Verdict = cls ? 'masked' : 'clear'; + const def = classDef(cls); + this.patchColumn(service, id, { + verdict, + class: cls, + tier: def?.tier ?? null, + source: 'admin', + decidedBy: null, + decidedAt: new Date().toISOString(), + }); + // Unmasking says so once it is saved, with an Undo that masks the + // column again with the class it had. + const undoTo = + !cls && shown.verdict !== 'clear' && shown.class ? shown.class : null; + this.api.decide(id, cls).subscribe({ + next: saved => { + if (!this.isLatest(id, n)) { + // A newer write is in flight; it owns the screen. Remember this + // one as confirmed (unless a newer one already confirmed) in case + // the newer one fails. + if (this.confirmed.has(id) && n > (this.confirmedSeq.get(id) ?? 0)) { + this.confirmed.set(id, { ...before, ...saved }); + this.confirmedSeq.set(id, n); + } + return; + } + this.confirmed.delete(id); + this.confirmedSeq.delete(id); + this.patchColumn(service, id, saved); + this.invalidateActivity(); + if (undoTo) this.offerUndo(col, undoTo); + }, + error: e => { + if (!this.isLatest(id, n)) return; + const back = this.confirmed.get(id) ?? before; + this.confirmed.delete(id); + this.confirmedSeq.delete(id); + this.patchColumn(service, id, back); + this.fail(e); + }, + }); + } + + private offerUndo(col: MaskedColumn, cls: string): void { + this.undoToast.open( + this.t('toast.notSensitive', { column: col.column }), + this.t('undo'), + () => { + const now = this.findColumn(col.service, col.id); + if (now && now.verdict === 'clear') this.decide(now, cls); + } + ); + } + + /** Last server-confirmed state of columns with writes in flight. */ + private confirmed = new Map(); + /** The newest write each confirmed state already includes. */ + private confirmedSeq = new Map(); + + private findColumn(service: string, id: string): MaskedColumn | undefined { + return this.columns.get(service)?.find(c => c.id === id); + } + + private patchColumn( + service: string, + id: string, + patch: Partial + ): void { + const list = this.columns.get(service); + if (!list) return; + const i = list.findIndex(c => c.id === id); + if (i < 0) return; + const prev = list[i]; + const next: MaskedColumn = { ...prev, ...patch }; + list[i] = next; + this.version++; + // Keep the rail summaries and header counts in step. + const svc = this.services.find(s => s.service === service); + const idx = svc?.tables.findIndex(t => t.name === next.table) ?? -1; + if (svc && idx >= 0) { + svc.tables[idx] = summarize( + next.table, + this.tableColumns(service, next.table), + svc.tables[idx].rows + ); + } + if (this.overview && prev.verdict !== next.verdict) { + const counts = { ...this.overview.counts }; + counts[prev.verdict]--; + counts[next.verdict]++; + this.overview = { ...this.overview, counts }; + } + } + + // --------------------------------------------------------------- access + + /** + * (Re)load access. The tab calls this every time it opens; a write in + * flight owns the screen, so a read that lands meanwhile is dropped. + */ + loadAccess(quiet = false): void { + this.failed.delete('access'); + this.api.access(quiet).subscribe({ + next: a => { + if (this.accessBusy || this.accessQueue.length) return; + this.access = a; + this.accessConfirmed = a; + }, + error: () => { + if (!this.access) this.failed.add('access'); + }, + }); + } + + /** Access as the server last confirmed it (with its version). */ + private accessConfirmed: AccessSettings | null = null; + private accessQueue: AccessWrite[] = []; + private accessBusy = false; + + /** + * Auto-save one access change. Writes go out one at a time, each with + * the version of the last answer, so a change made elsewhere since the + * page read access is refused (409) instead of being overwritten. A + * failure restores the last confirmed settings, drops the writes queued + * behind it (they were built on the refused state) and calls their + * `onError` so the view can drop local choices; a 409 also re-reads. + */ + updateAccess( + patch: AccessPatch, + optimistic: AccessSettings, + onError?: () => void, + toast = 'toast.access' + ): void { + this.access = optimistic; + this.accessQueue.push({ patch, onError, toast }); + this.drainAccess(); + } + + private drainAccess(): void { + if (this.accessBusy) return; + const write = this.accessQueue.shift(); + if (!write) return; + this.accessBusy = true; + const version = this.accessConfirmed?.version; + const body: AccessPatch = version + ? { ...write.patch, version } + : write.patch; + this.api.updateAccess(body).subscribe({ + next: a => { + this.accessBusy = false; + this.accessConfirmed = a; + if (!this.accessQueue.length) { + this.access = a; + this.invalidateActivity(); + this.refreshOverview(); + this.toast(write.toast); + } + this.drainAccess(); + }, + error: e => { + this.accessBusy = false; + const dropped = [write, ...this.accessQueue]; + this.accessQueue = []; + this.access = this.accessConfirmed; + dropped.forEach(w => w.onError?.()); + if (errorStatus(e) === 409) { + // Changed elsewhere (another admin, the CLI, a grant that ended): + // the page re-reads, so say that rather than "reload". + this.warn('toast.accessStale'); + this.loadAccess(true); + } else { + this.fail(e); + } + }, + }); + } + + private refreshOverview(): void { + const epoch = this.modeEpoch; + this.api.overview().subscribe({ + next: o => this.applyOverview(o, epoch), + error: () => undefined, + }); + } + + /** Baselines being tracked, for the grant limit they set (quiet). */ + trackedBaselines: string[] | null = null; + + loadTrackedBaselines(): void { + this.api.baselines().subscribe({ + next: list => + (this.trackedBaselines = list.filter(b => b.adopted).map(b => b.id)), + error: () => undefined, + }); + } + + // ---------------------------------------------------------------- masks + + loadMasks(): void { + this.failed.delete('masks'); + this.api.masks().subscribe({ + next: m => { + this.masks = m; + this.masksConfirmed = m; + }, + error: () => { + if (!this.masks) this.failed.add('masks'); + }, + }); + } + + private masksConfirmed: MaskSetting[] | null = null; + + setMask(cls: string, format: MaskFormat): void { + if (!this.masks) return; + const n = this.nextSeq('masks'); + this.masks = this.masks.map(m => (m.class === cls ? { ...m, format } : m)); + this.api.setMask(cls, format).subscribe({ + next: masks => { + if (masks.length) this.masksConfirmed = masks; + if (!this.isLatest('masks', n)) return; + if (masks.length) this.masks = masks; + this.invalidateActivity(); + // Column examples follow the class format. + this.loadColumns().subscribe(); + this.toast('toast.mask'); + }, + error: e => { + if (!this.isLatest('masks', n)) return; + this.masks = this.masksConfirmed; + this.fail(e); + }, + }); + } + + // ---------------------------------------------------------------- rules + + loadRules(): void { + this.failed.delete('rules'); + this.api.rules().subscribe({ + next: r => (this.rules = r), + error: () => { + if (!this.rules) this.failed.add('rules'); + }, + }); + if (!this.builtin) { + this.api.builtInRules().subscribe({ + next: b => (this.builtin = b), + error: () => undefined, + }); + } + } + + /** + * Save a custom rule. Returns the request so the dialog can stay open + * (and keep what was typed) until the server accepts it; a refused rule + * is the caller's to explain inline. Not being allowed to change + * settings (403) is no fault of the rule: it gets the error toast, like + * every other write. + */ + addRule( + rule: Pick + ): Observable { + return this.api.addRule(rule).pipe( + tap({ + next: saved => { + this.rules = [...(this.rules ?? []), saved]; + this.invalidateActivity(); + this.toast('toast.ruleAdded'); + this.reloadAfterRule(); + }, + error: e => { + if (errorStatus(e) === 403) this.fail(e); + }, + }) + ); + } + + /** What deleting `rule` leaves masked, from the loaded columns. */ + footprint(rule: DetectionRule): RuleFootprint { + return ruleFootprint(rule, this.allColumns()); + } + + /** + * Delete a rule. Its columns stay masked (automatic changes never clear + * masking); `unmask` lists the ones to mark not sensitive with it. A + * failure puts back only this rule, never the whole list. + */ + deleteRule(rule: DetectionRule, unmask: MaskedColumn[] = []): void { + const kept = this.footprint(rule).masked.length; + this.rules = (this.rules ?? []).filter(r => r.id !== rule.id); + this.api.deleteRule(rule.id).subscribe({ + next: () => { + this.invalidateActivity(); + if (!unmask.length) { + this.toast(kept ? 'toast.ruleDeletedKept' : 'toast.ruleDeleted', { + columns: this.plural('columns', kept), + }); + this.reloadAfterRule(); + return; + } + from(unmask) + .pipe( + mergeMap( + c => + this.api.decide(c.id, null).pipe( + map((): unknown => null), + catchError(e => of(e ?? true)) + ), + 4 + ), + toArray() + ) + .subscribe(results => { + const failures = results.filter(r => r !== null); + const done = results.length - failures.length; + const forbidden = failures.find(e => errorStatus(e) === 403); + if (!failures.length) { + this.toast('toast.ruleDeletedUnmasked', { + columns: this.plural('columns', done), + }); + } else if (forbidden) { + // Not allowed to mark columns not sensitive: the server says + // why; the reload shows which columns stayed masked. + this.fail(forbidden); + } else { + this.warn('toast.ruleUnmaskPartial', { + done, + total: results.length, + }); + } + this.reloadAfterRule(); + }); + }, + error: e => { + if (!(this.rules ?? []).some(r => r.id === rule.id)) { + this.rules = [...(this.rules ?? []), rule].sort( + (a, b) => a.id - b.id + ); + } + this.fail(e); + }, + }); + } + + /** Rules change verdicts and counts, so re-read them. */ + private reloadAfterRule(): void { + const epoch = this.modeEpoch; + forkJoin({ + overview: this.api.overview(), + services: this.api.tables(), + }).subscribe({ + next: ({ overview, services }) => { + this.applyOverview(overview, epoch); + this.setServices(services); + this.loadColumns().subscribe(); + }, + error: () => undefined, + }); + } + + // ------------------------------------------------------------- activity + + /** The kinds the Activity list shows ('' = every kind), null = never opened. */ + private activityKey: string | null = null; + private activityKinds: ActivityKind[] | null = null; + /** Pages already loaded per filter, so switching chips back is instant. */ + private activityCache = new Map< + string, + { events: ActivityEvent[]; hasMore: boolean } + >(); + private activitySub?: Subscription; + + /** + * Show activity of these kinds (all when null). The server filters and + * pages, so a quiet kind is never hidden behind a busy one. + */ + showActivity(kinds: ActivityKind[] | null): void { + const key = kinds?.join(',') ?? ''; + if (key === this.activityKey && this.activity) return; + this.activityKey = key; + this.activityKinds = kinds; + const cached = this.activityCache.get(key); + this.activity = cached?.events ?? null; + this.activityHasMore = cached?.hasMore ?? false; + this.loadActivity(); + } + + /** Loads (or refreshes) the first page. The old list stays up meanwhile. */ + loadActivity(): void { + if (this.activityKey === null) { + this.activityKey = ''; + this.activityKinds = null; + } + const key = this.activityKey; + this.failed.delete('activity'); + this.activitySub?.unsubscribe(); + this.activitySub = this.api + .activity({ kinds: this.activityKinds, limit: ACTIVITY_PAGE }) + .subscribe({ + next: page => { + // Older pages already loaded stay below the fresh first page. + const cur = key === this.activityKey ? this.activity : null; + const last = page.events[page.events.length - 1]; + const merged = + cur && last && cur.length > page.events.length + ? { + events: [...page.events, ...cur.filter(e => e.id < last.id)], + hasMore: this.activityHasMore, + } + : page; + this.activityCache.set(key, merged); + if (key !== this.activityKey) return; + this.activity = merged.events; + this.activityHasMore = merged.hasMore; + }, + error: () => { + if (key === this.activityKey && !this.activity) { + this.failed.add('activity'); + } + }, + }); + } + + /** Append the next older page. */ + loadOlderActivity(): void { + const list = this.activity; + const key = this.activityKey; + if (!list?.length || key === null || this.activityLoadingOlder) return; + this.activityLoadingOlder = true; + this.api + .activity({ + kinds: this.activityKinds, + limit: ACTIVITY_PAGE, + before: list[list.length - 1].id, + }) + .pipe(finalize(() => (this.activityLoadingOlder = false))) + .subscribe({ + next: page => { + if (key !== this.activityKey) return; + const seen = new Set(list.map(e => e.id)); + this.activity = [ + ...list, + ...page.events.filter(e => !seen.has(e.id)), + ]; + this.activityHasMore = page.hasMore; + this.activityCache.set(key, { + events: this.activity, + hasMore: page.hasMore, + }); + }, + error: e => this.fail(e), + }); + } + + /** + * Something happened that Activity would list. If the list was opened, + * refresh the current filter in place (other filters reload when picked); + * otherwise the tab loads fresh when opened. + */ + invalidateActivity(): void { + if (this.activityKey === null) return; + this.activityCache.clear(); + this.loadActivity(); + } + + // ---------------------------------------------------------- navigation + + /** Steer the Columns tab (from header counts or a rule's Matches). */ + showColumns(filter: ColumnFilter, idFilter: IdFilter | null = null): void { + this.filter = filter; + this.idFilter = idFilter; + this.viewReset$.next(); + } + + // -------------------------------------------------------------- helpers + + toast(key: string, params?: Record): void { + this.snackbar.openSnackBar(this.t(key, params), 'success'); + } + + warn(key: string, params?: Record): void { + this.snackbar.openSnackBar(this.t(key, params), 'warning'); + } + + fail(e: unknown): void { + this.snackbar.openError(isAppError(e) ? e : normalizeError(e)); + } + + ngOnDestroy(): void { + this.poll?.unsubscribe(); + this.stopWatching(); + this.activitySub?.unsubscribe(); + } +} diff --git a/src/app/adf-data-masking/data-masking.types.ts b/src/app/adf-data-masking/data-masking.types.ts new file mode 100644 index 000000000..12094ea93 --- /dev/null +++ b/src/app/adf-data-masking/data-masking.types.ts @@ -0,0 +1,257 @@ +/** + * Data Masking API types, as the UI sees them: responses pass through the + * case interceptor, so snake_case wire keys arrive camelCased + * (finished_at -> finishedAt, can_grant -> canGrant). + */ +import { MaskFormat, Tier } from './data-masking.catalog'; + +export type MaskingMode = 'off' | 'preview' | 'on'; +export type Verdict = 'masked' | 'review' | 'clear'; +export type StripMark = Tier | 'clear' | 'review'; + +/** A database service as the coverage lists name it. */ +export interface CoverageService { + service: string; + label: string; + /** Service type, e.g. "pgsql", "mongodb". */ + type: string; +} + +/** A database the last scan could not read. `reason` carries no data. */ +export interface ScanFailure { + serviceId?: number; + service: string; + label: string; + reason?: string | null; +} + +export type ScanOutcome = 'done' | 'partial' | 'failed'; + +export interface MaskingOverview { + mode: MaskingMode; + scan: { + status: 'idle' | 'running'; + progress?: number | null; + finishedAt?: string | null; + columns: number; + /** Outcome of the last finished scan; null before the first one. */ + lastStatus?: ScanOutcome | null; + /** Databases the last scan could not read. */ + failed?: ScanFailure[]; + /** + * The server could not read its scan history (a database error), so + * finishedAt, lastStatus and failed say nothing: "never scanned" is + * unknown, not true. + */ + unknown?: boolean; + }; + counts: { masked: number; review: number; clear: number }; + you: { + canGrant: boolean; + seesRealValues: boolean; + grantExpiresAt?: string | null; + }; + /** + * SQL services Data Masking covers, and database services it cannot mask + * (humans and apps are unaffected; AI sessions are blocked from them + * while masking is on). + */ + coverage?: { + covered: CoverageService[]; + uncovered: CoverageService[]; + }; +} + +export type EvidenceKind = 'name' | 'values' | 'context' | 'text' | 'penalty'; + +export interface EvidenceLine { + points: number; + kind: EvidenceKind; + text: string; +} + +export interface Evidence { + total: number; + thresholdMask: number; + thresholdReview: number; + lines: EvidenceLine[]; + /** [shape, percent] pairs, e.g. ["999-99-9999", 100]. */ + shapes?: [string, number][]; +} + +export interface MaskedColumn { + /** service.table.column */ + id: string; + service: string; + table: string; + column: string; + type: string; + verdict: Verdict; + class: string | null; + tier: Tier | null; + source: 'detected' | 'admin'; + decidedBy?: string | null; + decidedAt?: string | null; + confidence: number; + /** Masked rendering of a real value; never raw for masked/review. */ + example?: string | null; + /** Format `example` was rendered with. */ + format?: MaskFormat | null; + evidence: Evidence; +} + +export interface TableSummary { + name: string; + rows?: number | null; + columns: { masked: number; review: number; clear: number }; + /** One mark per column; review marks keep their tier for the outline. */ + strip: { column: string; mark: StripMark; tier?: Tier | null }[]; +} + +export interface ServiceTables { + service: string; + label: string; + /** + * When the service was last scanned; null = never. Tells a scanned + * database that holds no tables (`tables: []`) from one never scanned. + */ + scannedAt?: string | null; + /** Why the last scan could not read this database; null when it could. */ + scanError?: string | null; + /** The scan history could not be read: scannedAt null means unknown. */ + scanUnknown?: boolean; + tables: TableSummary[]; +} + +export type AdminAccessMode = 'none' | 'selected' | 'all'; +export type GrantDuration = '1h' | '8h' | '24h' | '7d' | 'forever'; + +export interface AdminGrant { + adminId: number; + name: string; + email: string; + /** null = until turned off. */ + expiresAt: string | null; +} + +export interface AdminRow { + adminId: number; + name: string; + email: string; + you?: boolean; +} + +export interface RoleAccess { + roleId: number; + name: string; + enabled: boolean; + scope: 'all' | 'except_restricted'; +} + +export interface AccessSettings { + admins: { + mode: AdminAccessMode; + grants: AdminGrant[]; + /** Every admin the super admin can pick from. */ + available: AdminRow[]; + }; + ai: 'masked' | 'same_as_admins'; + roles: RoleAccess[]; + /** + * Opaque version of these settings. PATCH sends it back; the server + * answers 409 when someone changed access since it was read. + */ + version?: string; +} + +/** + * PATCH /access body. `grants` is the complete desired list; a grant with a + * duration (new, or re-timed) gets a fresh expiry from the server, one + * without keeps the expiry it has. `roles` lists only the roles that change. + * `version` is added by the store from the last read. + */ +export interface AccessPatch { + admins?: { + mode?: AdminAccessMode; + grants?: { adminId: number; duration?: GrantDuration }[]; + }; + ai?: 'masked' | 'same_as_admins'; + roles?: { roleId: number; enabled: boolean; scope: RoleAccess['scope'] }[]; + version?: string; +} + +export interface MaskSetting { + class: string; + tier: Tier; + format: MaskFormat; + formatsAllowed: MaskFormat[]; +} + +export interface DetectionRule { + id: number; + match: 'name' | 'value'; + pattern: string; + class: string; + matches: number; + /** 'admin', or 'baseline:' for a rule a Compliance baseline added. */ + source?: string; +} + +export interface RuleTestResult { + matches: { id: string }[]; + /** Value rules sample live under a time budget; true when it ran out. */ + partial?: boolean; +} + +export type ActivityKind = + | 'mode' + | 'decision' + | 'access' + | 'scan' + | 'blocked' + | 'rule' + | 'baseline' + /** Aggregated "would mask" rows while the mode is Preview. */ + | 'preview'; + +export interface ActivityEvent { + id: number; + at: string; + actor: string; + kind: ActivityKind; + text: string; + target?: string | null; +} + +/** One page of activity, newest first. */ +export interface ActivityPage { + events: ActivityEvent[]; + /** Older events exist beyond this page (`before` the last id). */ + hasMore: boolean; +} + +export interface ActivityQuery { + /** Only these kinds; all kinds when absent. */ + kinds?: ActivityKind[] | null; + limit?: number; + /** Events older than this activity id. */ + before?: number; +} + +/** A Compliance baseline as the list endpoint returns it. */ +export interface BaselineSummary { + id: string; + name: string; + adopted: boolean; +} + +export interface Detector { + class: string; + detectors: string[]; +} + +export interface BuiltInRules { + ruleset: string; + version: string; + classes: Detector[]; +} diff --git a/src/app/adf-data-masking/df-class-picker/df-class-picker.component.html b/src/app/adf-data-masking/df-class-picker/df-class-picker.component.html new file mode 100644 index 000000000..399e09faa --- /dev/null +++ b/src/app/adf-data-masking/df-class-picker/df-class-picker.component.html @@ -0,0 +1,67 @@ + + + + + + + + + + + + + diff --git a/src/app/adf-data-masking/df-class-picker/df-class-picker.component.scss b/src/app/adf-data-masking/df-class-picker/df-class-picker.component.scss new file mode 100644 index 000000000..88b8eeab1 --- /dev/null +++ b/src/app/adf-data-masking/df-class-picker/df-class-picker.component.scss @@ -0,0 +1,170 @@ +@use '../dm-tier' as *; +@include dm-tiers; + +:host { + display: inline-flex; + max-width: 100%; + min-width: 0; +} + +.chip { + align-items: center; + background: var(--dm-tier-soft); + border: 1px solid transparent; + border-radius: 999px; + color: var(--dm-tier-ink); + cursor: pointer; + display: inline-flex; + font: inherit; + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + gap: 2px; + height: 24px; + max-width: 100%; + min-width: 0; + padding: 0 4px 0 10px; + transition: + background-color var(--df-duration-standard) var(--df-ease-standard), + border-color var(--df-duration-standard) var(--df-ease-standard), + color var(--df-duration-standard) var(--df-ease-standard); + + &:hover:not(:disabled) { + border-color: var(--dm-tier); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + &:disabled { + cursor: default; + opacity: 0.7; + } + + // Undecided: outlined, no wash. Matches the hollow strip tick. + &.is-review { + background: transparent; + border: 1px dashed var(--dm-tier); + } + + // Not sensitive: quiet text. The outline and caret come back on row + // hover or keyboard focus, so a column of clear rows is not a wall of + // identical chips. + &.is-clear { + background: transparent; + border-color: transparent; + color: var(--df-text-muted); + font-weight: var(--df-font-weight-body); + + .chip-caret { + opacity: 0; + } + + &:focus-visible { + border-color: var(--df-border); + + .chip-caret { + opacity: 0.7; + } + } + } +} + +:host-context(.row:hover) .chip.is-clear, +:host-context(.row:focus-within) .chip.is-clear { + border-color: var(--df-border); + color: var(--df-text-muted); + + .chip-caret { + opacity: 0.7; + } +} + +.chip-label { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.chip-lock { + font-size: 13px; + height: 13px; + margin: 0 2px 0 -3px; + width: 13px; +} + +.chip-caret { + font-size: 16px; + height: 16px; + opacity: 0.7; + width: 16px; +} + +// Menu content renders in the overlay but these classes are on our own +// template nodes, so component-scoped styles still apply. +.menu-group { + align-items: center; + color: var(--df-text-muted); + display: flex; + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + gap: var(--df-space-2); + padding: var(--df-space-3) var(--df-space-4) var(--df-space-1); +} + +.menu-dot { + background: var(--dm-tier-fill); + border-radius: 50%; + height: 8px; + width: 8px; +} + +.menu-item { + align-items: center; + display: flex; + font-size: var(--df-font-size-sm); + gap: var(--df-space-2); + justify-content: space-between; + width: 100%; +} + +.menu-check { + color: var(--df-accent); + font-size: 18px; + height: 18px; + margin: 0 !important; + width: 18px; +} + +.is-current .menu-item { + font-weight: var(--df-font-weight-medium); +} + +.menu-sep { + border-top: 1px solid var(--df-border-2); + margin: var(--df-space-1) 0; +} + +.mat-mdc-menu-item { + min-height: 34px; +} + +// The menu content uses theme tokens, so paint its panel with them too; +// otherwise dark/phosphor ink lands on Material's light overlay surface. +::ng-deep .mat-mdc-menu-panel.dm-class-menu { + --mat-menu-container-color: var(--df-surface); + background: var(--df-surface); + max-height: min(480px, calc(100vh - 32px)); +} + +.mat-mdc-menu-item { + --mat-menu-item-label-text-color: var(--df-text); + color: var(--df-text); + + &:hover:not([disabled]), + &.cdk-keyboard-focused, + &.cdk-program-focused { + background: var(--df-hover); + } +} diff --git a/src/app/adf-data-masking/df-class-picker/df-class-picker.component.ts b/src/app/adf-data-masking/df-class-picker/df-class-picker.component.ts new file mode 100644 index 000000000..682450a2c --- /dev/null +++ b/src/app/adf-data-masking/df-class-picker/df-class-picker.component.ts @@ -0,0 +1,67 @@ +import { + ChangeDetectionStrategy, + Component, + EventEmitter, + Input, + Output, + inject, +} from '@angular/core'; +import { NgClass, NgFor, NgIf } from '@angular/common'; +import { MatIconModule } from '@angular/material/icon'; +import { MatMenuModule } from '@angular/material/menu'; +import { TranslocoPipe, TranslocoService } from '@ngneat/transloco'; +import { classDef, classesByTier } from '../data-masking.catalog'; +import { Verdict } from '../data-masking.types'; + +/** + * df-class-picker: the class chip on a column row. Tier-tinted (restricted + * adds a lock so meaning is not color-only), hollow while the column needs + * review, neutral when not sensitive. Opens a menu grouped by tier with + * "Not sensitive" last. Emits the picked class id, or null. + */ +@Component({ + selector: 'df-class-picker', + standalone: true, + changeDetection: ChangeDetectionStrategy.OnPush, + imports: [NgIf, NgFor, NgClass, MatIconModule, MatMenuModule, TranslocoPipe], + templateUrl: './df-class-picker.component.html', + styleUrls: ['./df-class-picker.component.scss'], +}) +export class DfClassPickerComponent { + @Input() cls: string | null = null; + @Input() verdict: Verdict = 'clear'; + @Input() disabled = false; + /** Roving tabindex from the grid: -1 keeps the chip out of the Tab order. */ + @Input() tabIndex: number | null = null; + @Output() pick = new EventEmitter(); + + readonly groups = classesByTier(); + private transloco = inject(TranslocoService); + + /** + * The chip's text: the class, "Not sensitive" when clear, and "Choose a + * class" for a masked or review column without one (the server redacts + * it; "Not sensitive" there would say the opposite). + */ + get labelKey(): string { + if (this.verdict === 'clear') return 'dataMasking.notSensitive'; + return this.cls + ? `dataMasking.classes.${this.cls}` + : 'dataMasking.detection.pickClass'; + } + + get label(): string { + return this.transloco.translate(this.labelKey); + } + + get tier(): string { + return this.verdict === 'clear' + ? 'clear' + : (classDef(this.cls)?.tier ?? 'clear'); + } + + choose(cls: string | null): void { + if (cls === this.cls && this.verdict !== 'review') return; + this.pick.emit(cls); + } +} diff --git a/src/app/adf-data-masking/df-data-masking/df-data-masking.component.html b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.html new file mode 100644 index 000000000..61600f558 --- /dev/null +++ b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.html @@ -0,0 +1,130 @@ + + + +
    +
    + + + + + + {{ 'dataMasking.mode.' + m | transloco }} + + + +
    + + + + + + + +
    +
    + +

    + + {{ 'dataMasking.mode.locked' | transloco }} +

    + + + + + {{ + 'dataMasking.tabs.' + t | transloco + }} + + + + + + + + + + + + + + + + + + + + +
    +     diff --git a/src/app/adf-data-masking/df-data-masking/df-data-masking.component.scss b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.scss new file mode 100644 index 000000000..5290a6071 --- /dev/null +++ b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.scss @@ -0,0 +1,142 @@ +@use '../dm-controls' as *; +@include dm-lock; + +:host { + display: block; +} + +.dm { + margin: 0 auto; + max-width: var(--df-content-max); +} + +// The one control row: mode on the left, live counts beside it. +.dm-bar { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: var(--df-space-4); + margin-bottom: var(--df-space-2); +} + +.dm-mode { + --mat-standard-button-toggle-height: 32px; + font-size: var(--df-font-size-sm); + + ::ng-deep .mat-button-toggle-label-content { + line-height: 30px !important; + padding: 0 var(--df-space-4) !important; + } + + ::ng-deep .mat-button-toggle-checked { + font-weight: var(--df-font-weight-medium); + } + + // Keyboard focus: a full ring on the segment, not a thin bar. + ::ng-deep .mat-button-toggle.cdk-keyboard-focused { + border-radius: var(--df-radius-sm); + box-shadow: var(--df-focus-ring); + position: relative; + z-index: 1; + } +} + +// Same footprint as the toggle, so nothing shifts when the overview lands. +.dm-mode-skeleton { + background: var(--df-surface-2); + border-radius: var(--df-radius-sm); + display: inline-block; + height: 32px; + width: 212px; +} + +.lock { + border: 1px solid var(--df-border-2); + border-radius: var(--df-radius-sm); + margin-bottom: var(--df-space-3); + max-width: 560px; +} + +// Shown only while masking is on: a live dot on the segment in force. +.dm-mode-dot { + background: var(--df-success); + border-radius: 50%; + box-shadow: 0 0 0 3px var(--df-success-soft); + display: inline-block; + height: 7px; + margin-right: 6px; + vertical-align: 1px; + width: 7px; +} + +.dm-counts { + align-items: center; + color: var(--df-text-muted); + display: flex; + font-size: var(--df-font-size-sm); + gap: var(--df-space-2); +} + +.dm-count { + background: none; + border: 0; + border-radius: var(--df-radius-sm); + color: var(--df-text-2); + cursor: pointer; + font: inherit; + padding: 2px 4px; + + span { + color: var(--df-text); + font-weight: var(--df-font-weight-heading); + } + + &:hover { + background: var(--df-hover); + color: var(--df-text); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + // "To review" is a call to act, not a tier: it rides the accent, so it + // never reads as the amber identity tier. + &.is-review, + &.is-review span { + color: var(--df-accent-strong); + } +} + +.dm-sep { + color: var(--df-text-faint); +} + +.dm-tabs { + border-bottom: 1px solid var(--df-border-2); + margin-bottom: var(--df-space-5); + + // Nav-bar links keep the compiled light-theme indicator colour; paint it + // with the accent so the active tab reads in every theme. + ::ng-deep .mdc-tab-indicator__content--underline { + border-color: var(--df-accent) !important; + } +} + +// Phones: the tab bar gives way to one select (five labels do not fit). +.dm-tab-select { + display: none; + margin-bottom: var(--df-space-4); + width: 100%; +} + +@media (max-width: 600px) { + .dm-tabs { + display: none; + } + + .dm-tab-select { + display: block; + } +} diff --git a/src/app/adf-data-masking/df-data-masking/df-data-masking.component.spec.ts b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.spec.ts new file mode 100644 index 000000000..7d5d48558 --- /dev/null +++ b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.spec.ts @@ -0,0 +1,438 @@ +import { Component, Input } from '@angular/core'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { By } from '@angular/platform-browser'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { + ActivatedRoute, + Router, + convertToParamMap, + provideRouter, +} from '@angular/router'; +import { MatDialog } from '@angular/material/dialog'; +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { of } from 'rxjs'; +import { TranslocoService } from '@ngneat/transloco'; +import { DfDataMaskingComponent } from './df-data-masking.component'; +import { DfPaywallComponent } from 'src/app/shared/components/df-paywall/df-paywall.component'; +import { DataMaskingStore } from '../data-masking.store'; +import { dmTransloco } from '../dm-testing'; +import { MaskingMode } from '../data-masking.types'; + +function fakeStore(mode: MaskingMode = 'preview') { + return { + overview: { + mode, + scan: { status: 'idle', columns: 10 }, + counts: { masked: 167, review: 20, clear: 300 }, + you: { canGrant: true, seesRealValues: false }, + } as { + mode: MaskingMode; + scan: { status: string; columns: number }; + counts: { masked: number; review: number; clear: number }; + you: { canGrant: boolean; seesRealValues: boolean }; + } | null, + get mode() { + return this.overview?.mode ?? 'off'; + }, + get counts() { + return this.overview!.counts; + }, + get maskedTotal() { + return this.overview!.counts.masked + this.overview!.counts.review; + }, + get canLower() { + return !!this.overview?.you.canGrant; + }, + get foundColumns() { + const c = this.overview?.counts; + return ( + (!!c && c.masked + c.review + c.clear > 0) || + (this.overview?.scan.columns ?? 0) > 0 + ); + }, + unscanned: false, + scanned: true, + modeSaving: false, + failed: new Set(), + services: [{}, {}, {}], + maskedDatabases: 3, + loading: true, + loadFailed: false, + load: jest.fn(), + startWatching: jest.fn(), + stopWatching: jest.fn(), + setMode: jest.fn(), + showColumns: jest.fn(), + plural: (k: string, n: number) => `${n} ${k}`, + t: (k: string, p?: Record) => + p ? `${k}:${JSON.stringify(p)}` : k, + }; +} + +describe('DfDataMaskingComponent (mode)', () => { + let fixture: ComponentFixture; + let store: ReturnType; + let dialog: { open: jest.Mock }; + let closeWith: boolean; + + beforeEach(() => { + store = fakeStore(); + closeWith = true; + dialog = { + open: jest.fn(() => ({ afterClosed: () => of(closeWith) })), + }; + TestBed.configureTestingModule({ + imports: [ + DfDataMaskingComponent, + NoopAnimationsModule, + HttpClientTestingModule, + dmTransloco(), + ], + providers: [ + provideRouter([]), + { + provide: ActivatedRoute, + useValue: { + data: of({ data: 'ok' }), + queryParamMap: of(convertToParamMap({})), + }, + }, + ], + }); + TestBed.overrideComponent(DfDataMaskingComponent, { + set: { providers: [{ provide: DataMaskingStore, useValue: store }] }, + }); + TestBed.overrideProvider(MatDialog, { useValue: dialog }); + fixture = TestBed.createComponent(DfDataMaskingComponent); + fixture.detectChanges(); + }); + + it('turning on asks first: total masked, who (you included), what is left to review', () => { + fixture.componentInstance.requestMode('on'); + expect(dialog.open).toHaveBeenCalledTimes(1); + const data = dialog.open.mock.calls[0][1].data; + expect(data.title).toBe('mode.confirmOn.title'); + // When On, review columns are masked too: the total, once. + expect(data.message).toContain('mode.confirmOn.message:'); + expect(data.message).toContain('"columns":"187 columns"'); + expect(data.message).toContain('"databases":"3 databases"'); + expect(data.message).toContain('mode.confirmOn.review:{"review":20}'); + expect(store.setMode).toHaveBeenCalledWith('on'); + }); + + it('the turn-on confirm drops "you included" for an admin with access', () => { + store.overview!.you.seesRealValues = true; + store.overview!.counts.review = 0; + fixture.componentInstance.requestMode('on'); + const data = dialog.open.mock.calls[0][1].data; + expect(data.message).toContain('mode.confirmOn.messageWithAccess:'); + expect(data.message).not.toContain('mode.confirmOn.review'); + }); + + it('the real copy is short and says admins are masked too', () => { + const real = TestBed.inject(TranslocoService); + const msg = real.translate('dataMasking.mode.confirmOn.message', { + columns: '187 columns', + databases: '3 databases', + }); + expect(msg).toBe( + '187 columns in 3 databases will be masked for apps, AI tools and admins, you included.' + ); + }); + + it('leaving On for Preview asks first (masking stops)', () => { + store.overview!.mode = 'on'; + fixture.componentInstance.requestMode('preview'); + expect(store.setMode).toHaveBeenCalledWith('preview'); + expect(dialog.open).toHaveBeenCalledTimes(1); + const data = dialog.open.mock.calls[0][1].data; + expect(data.title).toBe('mode.confirmPreview.title'); + expect(data.danger).toBe(true); + }); + + it('a cancelled On to Preview changes nothing', () => { + closeWith = false; + store.overview!.mode = 'on'; + fixture.componentInstance.requestMode('preview'); + expect(store.setMode).not.toHaveBeenCalled(); + }); + + it('Preview to Off needs no dialog (nothing was masked)', () => { + store.overview!.mode = 'preview'; + fixture.componentInstance.requestMode('off'); + expect(dialog.open).not.toHaveBeenCalled(); + expect(store.setMode).toHaveBeenCalledWith('off'); + }); + + it('the toggle never shows a mode before it is confirmed', () => { + closeWith = false; + const el: HTMLElement = fixture.nativeElement; + const btn = (m: string) => + el.querySelector(`[data-testid="dm-mode-${m}"] button`) as HTMLElement; + jest.useFakeTimers(); + btn('on').click(); + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + fixture.detectChanges(); + // Back to the mode in force while the dialog is open, and after cancel. + expect(fixture.componentInstance.modeGroup!.value).toBe('preview'); + expect(btn('preview').getAttribute('aria-pressed')).toBe('true'); + expect(btn('on').getAttribute('aria-pressed')).toBe('false'); + expect(store.setMode).not.toHaveBeenCalled(); + }); + + it('ignores mode requests while a mode change is saving', () => { + store.modeSaving = true; + store.overview!.mode = 'off'; + fixture.componentInstance.requestMode('preview'); + expect(store.setMode).not.toHaveBeenCalled(); + }); + + it('turning off from On is a danger confirm', () => { + store.overview!.mode = 'on'; + fixture.componentInstance.requestMode('off'); + const data = dialog.open.mock.calls[0][1].data; + expect(data.danger).toBe(true); + expect(store.setMode).toHaveBeenCalledWith('off'); + }); + + it('Off to Preview needs no dialog', () => { + store.overview!.mode = 'off'; + fixture.componentInstance.requestMode('preview'); + expect(dialog.open).not.toHaveBeenCalled(); + expect(store.setMode).toHaveBeenCalledWith('preview'); + }); + + it('header counts jump to the matching Columns filter', () => { + const btn: HTMLButtonElement = fixture.nativeElement.querySelector( + '[data-testid="dm-count-review"]' + ); + expect(btn.textContent).toContain('20'); + btn.click(); + expect(store.showColumns).toHaveBeenCalledWith('review'); + }); + + it('header counts are disjoint: masked, then to review', () => { + const el: HTMLElement = fixture.nativeElement; + const masked = el.querySelector('[data-testid="dm-count-masked"]')!; + expect(masked.textContent).toContain('167'); + expect( + el.querySelector('[data-testid="dm-count-review"]')!.textContent + ).toContain('20'); + }); + + it('while the overview loads, no mode is shown as selected', () => { + store.overview = null; + fixture.detectChanges(); + const el: HTMLElement = fixture.nativeElement; + expect(el.querySelector('[data-testid="dm-mode"]')).toBeNull(); + expect(el.querySelector('[data-testid="dm-mode-loading"]')).toBeTruthy(); + }); + + it('before any scan with masking off, the page leads with the scan, not the mode', () => { + store.unscanned = true; + store.overview!.mode = 'off'; + fixture.detectChanges(); + const el: HTMLElement = fixture.nativeElement; + expect(el.querySelector('[data-testid="dm-count-masked"]')).toBeNull(); + expect(el.querySelector('[data-testid="dm-mode"]')).toBeNull(); + }); + + it('before any scan with masking left on, Off stays reachable; Preview and On wait', () => { + store.unscanned = true; + store.scanned = false; + store.overview!.mode = 'on'; + fixture.detectChanges(); + const el: HTMLElement = fixture.nativeElement; + const btn = (m: string) => + el.querySelector( + `[data-testid="dm-mode-${m}"] button` + ) as HTMLButtonElement; + expect(btn('off').disabled).toBe(false); + expect(btn('preview').disabled).toBe(true); + fixture.componentInstance.requestMode('preview'); + expect(store.setMode).not.toHaveBeenCalled(); + }); + + it('while the first scan runs (columns arriving, none finished), Preview and On still wait', () => { + // Live, the first columns land before the scan ends: the counts show, + // but the server refuses On until a scan has finished. + store.unscanned = false; + store.scanned = false; + store.overview!.mode = 'off'; + store.overview!.scan = { status: 'running', columns: 12 }; + fixture.detectChanges(); + const el: HTMLElement = fixture.nativeElement; + const btn = (m: string) => + el.querySelector( + `[data-testid="dm-mode-${m}"] button` + ) as HTMLButtonElement; + expect(btn('off').disabled).toBe(false); + expect(btn('preview').disabled).toBe(true); + expect(btn('on').disabled).toBe(true); + fixture.componentInstance.requestMode('on'); + expect(dialog.open).not.toHaveBeenCalled(); + expect(store.setMode).not.toHaveBeenCalled(); + // Once it has finished, both are offered. + store.scanned = true; + fixture.detectChanges(); + expect(btn('preview').disabled).toBe(false); + expect(btn('on').disabled).toBe(false); + }); + + it('a finished scan that found no columns counts as no scan: no mode control, and On waits', () => { + // Default install: `df:masking setup` scanned only the empty built-in + // db. The server refuses On (409) until a scan finds columns. + store.unscanned = false; + store.scanned = true; + store.overview!.mode = 'off'; + store.overview!.scan = { status: 'idle', columns: 0 }; + store.overview!.counts = { masked: 0, review: 0, clear: 0 }; + fixture.detectChanges(); + const el: HTMLElement = fixture.nativeElement; + expect(el.querySelector('[data-testid="dm-mode"]')).toBeNull(); + expect(el.querySelector('[data-testid="dm-count-masked"]')).toBeNull(); + // A mode set by other means (Apply sets Preview): Off stays reachable. + store.overview!.mode = 'preview'; + fixture.detectChanges(); + const btn = (m: string) => + el.querySelector( + `[data-testid="dm-mode-${m}"] button` + ) as HTMLButtonElement; + expect(btn('off').disabled).toBe(false); + expect(btn('on').disabled).toBe(true); + fixture.componentInstance.requestMode('on'); + expect(dialog.open).not.toHaveBeenCalled(); + expect(store.setMode).not.toHaveBeenCalled(); + // The next scan finds columns: On is offered. + store.overview!.scan = { status: 'idle', columns: 12 }; + store.overview!.counts = { masked: 2, review: 0, clear: 10 }; + fixture.detectChanges(); + expect(btn('on').disabled).toBe(false); + }); + + it('with masking On, an admin who cannot lower it sees Off and Preview locked', () => { + store.overview!.mode = 'on'; + store.overview!.you.canGrant = false; + fixture.detectChanges(); + const el: HTMLElement = fixture.nativeElement; + const btn = (m: string) => + el.querySelector( + `[data-testid="dm-mode-${m}"] button` + ) as HTMLButtonElement; + expect(btn('off').disabled).toBe(true); + expect(btn('preview').disabled).toBe(true); + expect(btn('on').disabled).toBe(false); + expect( + el.querySelector('[data-testid="dm-mode-lock"]')!.textContent + ).toContain('Only the super admin can lower the masking mode.'); + fixture.componentInstance.requestMode('off'); + expect(dialog.open).not.toHaveBeenCalled(); + // In Preview the same admin may raise it again. + store.overview!.mode = 'preview'; + fixture.detectChanges(); + expect(el.querySelector('[data-testid="dm-mode-lock"]')).toBeNull(); + expect(btn('on').disabled).toBe(false); + }); +}); + +describe('DfDataMaskingComponent (?filter= deep link)', () => { + function create(loading: boolean) { + // loadFailed keeps the tab body out: this is about the shell only. + const store = { + ...fakeStore(), + loading, + loadFailed: true, + filter: null as string | null, + }; + TestBed.configureTestingModule({ + imports: [ + DfDataMaskingComponent, + NoopAnimationsModule, + HttpClientTestingModule, + dmTransloco(), + ], + providers: [ + provideRouter([]), + { + provide: ActivatedRoute, + useValue: { + data: of({ data: 'ok' }), + queryParamMap: of(convertToParamMap({ filter: 'review' })), + }, + }, + ], + }); + TestBed.overrideComponent(DfDataMaskingComponent, { + set: { providers: [{ provide: DataMaskingStore, useValue: store }] }, + }); + const router = TestBed.inject(Router); + const nav = jest.spyOn(router, 'navigate').mockResolvedValue(true); + const fixture = TestBed.createComponent(DfDataMaskingComponent); + fixture.detectChanges(); + return { store, nav }; + } + + it('opens Columns on the filter once loaded, then drops it from the URL', () => { + const { store, nav } = create(false); + expect(store.showColumns).toHaveBeenCalledWith('review'); + expect(nav).toHaveBeenCalledWith( + [], + expect.objectContaining({ queryParams: { filter: null } }) + ); + }); + + it('hands the filter to the first Columns view while loading', () => { + const { store } = create(true); + expect(store.filter).toBe('review'); + expect(store.showColumns).not.toHaveBeenCalled(); + }); +}); + +/** The shared paywall tracks a hit and boots Calendly: stubbed here. */ +@Component({ selector: 'df-paywall', standalone: true, template: '' }) +class StubPaywallComponent { + @Input() serviceName = ''; +} + +describe('DfDataMaskingComponent (paywall)', () => { + it('without the data_masking resource shows the shared paywall, on any license', () => { + const store = fakeStore(); + TestBed.configureTestingModule({ + imports: [ + DfDataMaskingComponent, + NoopAnimationsModule, + HttpClientTestingModule, + dmTransloco(), + ], + providers: [ + provideRouter([]), + { + provide: ActivatedRoute, + useValue: { + data: of({ data: 'paywall' }), + queryParamMap: of(convertToParamMap({})), + }, + }, + ], + }); + TestBed.overrideComponent(DfDataMaskingComponent, { + remove: { imports: [DfPaywallComponent], providers: [DataMaskingStore] }, + add: { + imports: [StubPaywallComponent], + providers: [{ provide: DataMaskingStore, useValue: store }], + }, + }); + const fixture = TestBed.createComponent(DfDataMaskingComponent); + fixture.detectChanges(); + const paywall = fixture.debugElement.query( + By.directive(StubPaywallComponent) + ); + expect(paywall).toBeTruthy(); + expect(paywall.componentInstance.serviceName).toBe('Data Masking'); + const el = fixture.nativeElement as HTMLElement; + expect(el.querySelector('.dm')).toBeNull(); + expect(store.load).not.toHaveBeenCalled(); + expect(store.startWatching).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/adf-data-masking/df-data-masking/df-data-masking.component.ts b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.ts new file mode 100644 index 000000000..4748cc8de --- /dev/null +++ b/src/app/adf-data-masking/df-data-masking/df-data-masking.component.ts @@ -0,0 +1,270 @@ +import { + ChangeDetectorRef, + Component, + OnDestroy, + OnInit, + ViewChild, + inject, +} from '@angular/core'; +import { NgFor, NgIf, NgSwitch, NgSwitchCase } from '@angular/common'; +import { ActivatedRoute, Router, RouterLink } from '@angular/router'; +import { + MatButtonToggleGroup, + MatButtonToggleModule, +} from '@angular/material/button-toggle'; +import { MatDialog, MatDialogModule } from '@angular/material/dialog'; +import { MatIconModule } from '@angular/material/icon'; +import { MatSelectModule } from '@angular/material/select'; +import { MatTabsModule } from '@angular/material/tabs'; +import { TranslocoPipe } from '@ngneat/transloco'; +import { Subscription } from 'rxjs'; +import { DfPaywallComponent } from 'src/app/shared/components/df-paywall/df-paywall.component'; +import { DfSkeletonComponent } from 'src/app/shared/components/df-skeleton/df-skeleton.component'; +import { DfEmptyStateComponent } from 'src/app/shared/components/df-empty-state/df-empty-state.component'; +import { ColumnFilter, DataMaskingStore } from '../data-masking.store'; +import { MaskingMode } from '../data-masking.types'; +import { DfDmColumnsComponent } from '../df-dm-columns/df-dm-columns.component'; +import { DfDmAccessComponent } from '../df-dm-access/df-dm-access.component'; +import { DfDmMasksComponent } from '../df-dm-masks/df-dm-masks.component'; +import { DfDmDetectionComponent } from '../df-dm-detection/df-dm-detection.component'; +import { DfDmActivityComponent } from '../df-dm-activity/df-dm-activity.component'; +import { + DfDmConfirmDialogComponent, + DmConfirmData, +} from './df-dm-confirm-dialog.component'; + +export type DmTab = 'columns' | 'access' | 'masks' | 'detection' | 'activity'; +const COLUMN_FILTERS: ColumnFilter[] = ['review', 'masked', 'clear', 'all']; +export const DM_TABS: DmTab[] = [ + 'columns', + 'access', + 'masks', + 'detection', + 'activity', +]; + +/** + * Data Masking page shell (/api-security/data-masking). The admin chrome + * already prints the page H1, so the shell carries only the one primary + * control (Off / Preview / On), live counts that double as navigation, and + * the tab bar. The tab lives in ?tab= so links and reloads keep it. + */ +@Component({ + selector: 'df-data-masking', + standalone: true, + imports: [ + NgFor, + NgIf, + NgSwitch, + NgSwitchCase, + MatButtonToggleModule, + MatDialogModule, + MatIconModule, + MatSelectModule, + MatTabsModule, + RouterLink, + TranslocoPipe, + DfPaywallComponent, + DfSkeletonComponent, + DfEmptyStateComponent, + DfDmColumnsComponent, + DfDmAccessComponent, + DfDmMasksComponent, + DfDmDetectionComponent, + DfDmActivityComponent, + ], + providers: [DataMaskingStore], + templateUrl: './df-data-masking.component.html', + styleUrls: ['./df-data-masking.component.scss'], +}) +export class DfDataMaskingComponent implements OnInit, OnDestroy { + readonly store = inject(DataMaskingStore); + private route = inject(ActivatedRoute); + private router = inject(Router); + private dialog = inject(MatDialog); + private cdr = inject(ChangeDetectorRef); + + @ViewChild('modeGroup') modeGroup?: MatButtonToggleGroup; + + readonly tabs = DM_TABS; + readonly modes: MaskingMode[] = ['off', 'preview', 'on']; + paywall = false; + tab: DmTab = 'columns'; + private subs: Subscription[] = []; + + ngOnInit(): void { + this.subs.push( + this.route.data.subscribe(d => { + this.paywall = d?.['data'] === 'paywall'; + if (this.paywall) return; + if (!this.store.overview) this.store.load(); + // Pick up scans that finish while the page is open. + this.store.startWatching(); + }), + this.route.queryParamMap.subscribe(q => { + const t = q.get('tab') as DmTab | null; + this.tab = t && DM_TABS.includes(t) ? t : 'columns'; + this.consumeFilter(q.get('filter')); + }) + ); + } + + ngOnDestroy(): void { + this.subs.forEach(s => s.unsubscribe()); + this.store.stopWatching(); + } + + /** + * Before the first scan with masking off there is nothing to switch: the + * page leads with its one action (scan) and the control stays out of the + * way. A scan that found no columns (empty databases) counts as no scan. + * A mode left on by other means still shows, so Off is reachable. + */ + get showModeControl(): boolean { + const s = this.store; + return ( + !!s.overview && !((s.unscanned || !s.foundColumns) && s.mode === 'off') + ); + } + + /** + * Lowering the mode from On (to Preview or Off) is the super admin's call + * once one is designated; raising it is always allowed. + */ + get lowerLocked(): boolean { + return this.store.mode === 'on' && !this.store.canLower; + } + + /** + * Off is never held back by the scan state. Preview and On wait for the + * first scan that finds columns (there is nothing to mask yet, and the + * server refuses On until then), including while it runs and its first + * columns arrive; Off and Preview lock while On when this admin cannot + * lower it. + */ + modeDisabled(m: MaskingMode): boolean { + if (m === this.store.mode) return false; + if (m !== 'off' && !(this.store.scanned && this.store.foundColumns)) + return true; + return m !== 'on' && this.lowerLocked; + } + + /** + * ?filter=review|masked|clear|all opens Columns on that filter (deep + * links such as a Compliance check's Fix). Applied once, then dropped + * from the URL so later tab switches never re-impose it. + */ + private consumeFilter(f: string | null): void { + if (!f || !COLUMN_FILTERS.includes(f as ColumnFilter)) return; + if (this.store.loading || !this.store.overview) { + // Read by the Columns tab when its first view is built. + this.store.filter = f as ColumnFilter; + } else { + this.store.showColumns(f as ColumnFilter); + } + this.router.navigate([], { + relativeTo: this.route, + queryParams: { filter: null }, + queryParamsHandling: 'merge', + replaceUrl: true, + }); + } + + tabQuery(tab: DmTab): Record { + return { tab: tab === 'columns' ? null : tab }; + } + + setTab(tab: DmTab): void { + this.router.navigate([], { + relativeTo: this.route, + queryParams: { tab: tab === 'columns' ? null : tab }, + queryParamsHandling: 'merge', + replaceUrl: true, + }); + } + + /** Header count jump: open Columns on that filter. */ + showColumns(filter: 'review' | 'masked'): void { + this.store.showColumns(filter); + this.setTab('columns'); + } + + requestMode(mode: MaskingMode): void { + // The toggle shows the mode in force, never the one asked for: put it + // back now, and let the store move it once the change is real. + this.syncMode(); + if (mode === this.store.mode || this.store.modeSaving) return; + if (this.modeDisabled(mode)) return; + const data = this.confirmFor(mode); + if (!data) { + this.store.setMode(mode); + return; + } + this.dialog + .open(DfDmConfirmDialogComponent, { data, autoFocus: 'dialog' }) + .afterClosed() + .subscribe(ok => { + if (ok) this.store.setMode(mode); + this.syncMode(); + }); + } + + /** + * A confirm whenever masking is added or removed: turning On, and leaving + * On (for Preview or Off). Off and Preview swap without one: neither + * masks anything. + */ + private confirmFor(mode: MaskingMode): DmConfirmData | null { + const s = this.store; + const t = (k: string, p?: Record) => s.t(k, p); + if (mode === 'on') { + const o = s.overview; + const review = o?.counts.review ?? 0; + const audience = o?.you.seesRealValues + ? 'mode.confirmOn.messageWithAccess' + : 'mode.confirmOn.message'; + const message = [ + t(audience, { + columns: s.plural('columns', s.maskedTotal), + databases: s.plural('databases', s.maskedDatabases), + }), + review > 0 ? t('mode.confirmOn.review', { review }) : '', + ] + .filter(Boolean) + .join(' '); + return { + title: t('mode.confirmOn.title'), + message, + confirm: t('mode.confirmOn.confirm'), + cancel: t('cancel'), + }; + } + if (s.mode !== 'on') return null; + const key = mode === 'preview' ? 'confirmPreview' : 'confirmOff'; + return { + title: t(`mode.${key}.title`), + message: t(`mode.${key}.message`), + confirm: t(`mode.${key}.confirm`), + cancel: t('cancel'), + danger: true, + }; + } + + /** + * mat-button-toggle selects the clicked toggle itself, after our (change) + * handler runs, so reset both now and once its click handling is done. + */ + private syncMode(): void { + const reset = () => { + const group = this.modeGroup; + if (!group) return; + group.value = this.store.mode; + group._buttonToggles?.forEach( + b => (b.checked = b.value === this.store.mode) + ); + this.cdr.markForCheck(); + }; + reset(); + setTimeout(reset); + } +} diff --git a/src/app/adf-data-masking/df-data-masking/df-dm-confirm-dialog.component.ts b/src/app/adf-data-masking/df-data-masking/df-dm-confirm-dialog.component.ts new file mode 100644 index 000000000..7ede4a5fc --- /dev/null +++ b/src/app/adf-data-masking/df-data-masking/df-dm-confirm-dialog.component.ts @@ -0,0 +1,77 @@ +import { Component, Inject } from '@angular/core'; +import { MatButtonModule } from '@angular/material/button'; +import { NgIf } from '@angular/common'; +import { MAT_DIALOG_DATA, MatDialogModule } from '@angular/material/dialog'; + +export interface DmConfirmData { + title: string; + message: string; + confirm: string; + cancel: string; + danger?: boolean; + /** + * A second, alternative confirm (e.g. "Delete and unmask 3"). The dialog + * then closes with 'extra'. + */ + extra?: string; +} + +/** Confirm: one sentence carrying the real numbers. */ +@Component({ + selector: 'df-dm-confirm-dialog', + standalone: true, + imports: [NgIf, MatDialogModule, MatButtonModule], + template: ` +

    {{ data.title }}

    +
    {{ data.message }}
    +
    + + + +
    + `, + styles: [ + ` + @use '../../shared/components/df-dialog-surface' as *; + @include df-dialog-surface; + + [mat-dialog-content] { + color: var(--df-text-2); + font-size: var(--df-font-size-sm); + max-width: 420px; + } + + .confirm.danger { + --mdc-filled-button-container-color: var(--df-danger-soft); + --mdc-filled-button-label-text-color: var(--df-danger); + border: 1px solid var(--df-danger-border); + } + `, + ], +}) +export class DfDmConfirmDialogComponent { + constructor(@Inject(MAT_DIALOG_DATA) public data: DmConfirmData) {} +} diff --git a/src/app/adf-data-masking/df-dm-access/df-dm-access.component.html b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.html new file mode 100644 index 000000000..0ee63d52c --- /dev/null +++ b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.html @@ -0,0 +1,189 @@ + + + + + + +
    +

    {{ 'dataMasking.access.title' | transloco }}

    + +

    + + {{ 'dataMasking.access.locked' | transloco }} +

    + + +
    +
    + + {{ + 'dataMasking.access.admins' | transloco + }} +
    +
    + + {{ + 'dataMasking.access.none' | transloco + }} + {{ 'dataMasking.access.selected' | transloco }} + {{ + 'dataMasking.access.allAdmins' | transloco + }} + + +
      +
    • + + {{ ad.name }} + {{ + 'dataMasking.access.you' | transloco + }} + + + + {{ + 'dataMasking.access.durations.' + durationOf(ad) + | transloco + }}warning_amber + {{ 'dataMasking.access.durations.' + d | transloco + }}{{ + 'dataMasking.access.exceeds' | transloco: { name: b } + }} + + {{ + 'dataMasking.access.until' | transloco: { when: u } + }} + +
      • +
    +
    +
    + + +
    +
    + + {{ 'dataMasking.access.ai' | transloco }} +
    +
    + + {{ + 'dataMasking.access.aiMasked' | transloco + }} + {{ 'dataMasking.access.aiSame' | transloco }} + +
    +
    + + +
    +
    + + {{ 'dataMasking.access.roles' | transloco }} +
    +
    +
      +
    • + + {{ r.name }} + + + {{ + (r.scope === 'all' + ? 'dataMasking.access.scopeAll' + : 'dataMasking.access.scopeExceptRestricted' + ) | transloco + }} + {{ + 'dataMasking.access.scopeAll' | transloco + }} + {{ + 'dataMasking.access.scopeExceptRestricted' | transloco + }} + +
      • +
    + + {{ + 'dataMasking.access.noRoles' | transloco + }} + +
    +
    +
    +     diff --git a/src/app/adf-data-masking/df-dm-access/df-dm-access.component.scss b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.scss new file mode 100644 index 000000000..02b2270af --- /dev/null +++ b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.scss @@ -0,0 +1,196 @@ +@use '../dm-controls' as *; +@include dm-select; +@include dm-segmented; +@include dm-lock; + +:host { + display: block; +} + +// Flat like the other tabs: a section title, then the principals. +.card { + max-width: 860px; +} + +.card-title { + border-bottom: 1px solid var(--df-border-2); + color: var(--df-text); + font-size: var(--df-font-size-lg); + font-weight: var(--df-font-weight-heading); + margin: 0; + padding: 0 0 var(--df-space-3); +} + +// A table of principals: who on the left, how on the right. +.principal { + display: grid; + gap: var(--df-space-4); + grid-template-columns: 200px minmax(0, 1fr); + padding: var(--df-space-4) 0; + + & + & { + border-top: 1px solid var(--df-border-2); + } +} + +.who { + align-items: center; + color: var(--df-text); + display: flex; + font-size: var(--df-font-size-sm); + font-weight: var(--df-font-weight-medium); + gap: var(--df-space-2); + height: 30px; + + mat-icon { + color: var(--df-text-muted); + font-size: 18px; + height: 18px; + width: 18px; + } +} + +.how { + max-width: 620px; + min-width: 0; +} + +// Admin and role rows share one track (name | select | until), and both +// lists carry the same 1px frame, so their selects line up. +.admins, +.roles { + border: 1px solid transparent; + list-style: none; + margin: 0; + padding: 0; +} + +.admins { + border-color: var(--df-border-2); + border-radius: var(--df-radius-sm); + margin-top: var(--df-space-3); +} + +.admin, +.role { + align-items: center; + column-gap: var(--df-space-3); + display: grid; + grid-template-columns: minmax(0, 1fr) 176px 96px; + min-height: var(--df-row-height); + padding: 0 var(--df-space-3) 0 var(--df-space-1); +} + +.admin { + & + & { + border-top: 1px solid var(--df-border-2); + } + + mat-checkbox { + min-width: 0; + } +} + +.admin-name { + color: var(--df-text); + font-size: var(--df-font-size-sm); +} + +// Locked for this admin (only the super admin can grant): reads as such. +.mat-mdc-checkbox-disabled .admin-name { + color: var(--df-text-muted); +} + +.you { + color: var(--df-text-muted); + font-size: var(--df-font-size-sm); + margin-left: var(--df-space-1); +} + +.until { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + grid-column: 3; +} + +.dm-select { + grid-column: 2; +} + +.role { + mat-slide-toggle { + font-size: var(--df-font-size-sm); + min-width: 0; + + ::ng-deep .mdc-form-field > label { + padding-left: var(--df-space-3); + } + } +} + +.none { + color: var(--df-text-muted); + font-size: var(--df-font-size-sm); + line-height: 30px; +} + +@media (max-width: 760px) { + .principal { + grid-template-columns: minmax(0, 1fr); + gap: var(--df-space-2); + padding: var(--df-space-3) 0; + } + + .admin, + .role { + grid-template-columns: minmax(0, 1fr) 150px; + } + + .until { + grid-column: 1 / -1; + padding: 0 0 var(--df-space-2) var(--df-space-6); + } +} + +// Duration options stay on one line ("Until turned off Exceeds HIPAA"); +// the panel grows to fit them (durationPanelWidth). +.duration-option ::ng-deep .mdc-list-item__primary-text { + white-space: nowrap; +} + +// A duration longer than a tracked baseline allows: named, in the option. +.exceeds { + color: var(--df-warning, var(--df-danger)); + font-size: var(--df-font-size-xs); + margin-left: var(--df-space-2); +} + +.exceeds-icon { + color: var(--df-warning, var(--df-danger)); + font-size: 16px; + height: 16px; + margin-left: var(--df-space-1); + vertical-align: -3px; + width: 16px; +} + +// Phones: the select drops under the name instead of spilling out. +@media (max-width: 420px) { + .admin, + .role { + grid-template-columns: minmax(0, 1fr); + padding-bottom: var(--df-space-2); + row-gap: var(--df-space-1); + } + + .dm-select { + grid-column: 1; + margin-left: var(--df-space-6); + max-width: calc(100% - var(--df-space-6)); + } + + .until { + grid-column: 1; + padding-bottom: 0; + } +} diff --git a/src/app/adf-data-masking/df-dm-access/df-dm-access.component.spec.ts b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.spec.ts new file mode 100644 index 000000000..6b55b44f7 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.spec.ts @@ -0,0 +1,293 @@ +import { + ComponentFixture, + TestBed, + fakeAsync, + tick, +} from '@angular/core/testing'; +import { MatDialog } from '@angular/material/dialog'; +import { MatSelect } from '@angular/material/select'; +import { By } from '@angular/platform-browser'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { of } from 'rxjs'; +import { DfDmAccessComponent, liveGrants } from './df-dm-access.component'; +import { DataMaskingStore } from '../data-masking.store'; +import { DfDataMaskingApiService } from '../data-masking-api.service'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { dmTransloco } from '../dm-testing'; +import { AccessSettings } from '../data-masking.types'; + +const HOUR = 3_600_000; +const inHours = (h: number) => new Date(Date.now() + h * HOUR).toISOString(); + +function settings(over: Partial = {}): AccessSettings { + return { + admins: { + mode: 'selected', + grants: [ + { adminId: 2, name: 'Priya Shah', email: 'p@x', expiresAt: inHours(4) }, + ], + available: [ + { adminId: 1, name: 'Kevin Admin', email: 'k@x' }, + { adminId: 2, name: 'Priya Shah', email: 'p@x', you: true }, + { adminId: 3, name: 'Sam Lee', email: 's@x' }, + ], + }, + ai: 'masked', + roles: [ + { roleId: 1, name: 'support', enabled: true, scope: 'except_restricted' }, + { + roleId: 2, + name: 'billing', + enabled: false, + scope: 'except_restricted', + }, + ], + ...over, + }; +} + +describe('DfDmAccessComponent', () => { + let fixture: ComponentFixture; + let store: DataMaskingStore; + let api: { + updateAccess: jest.Mock; + access: jest.Mock; + overview: jest.Mock; + baselines: jest.Mock; + }; + let dialog: { open: jest.Mock }; + let closeWith: boolean; + + const el = () => fixture.nativeElement as HTMLElement; + const toggle = (group: string, label: string) => + Array.from( + el().querySelectorAll(`[data-testid="${group}"] mat-button-toggle`) + ).find(t => t.textContent?.trim() === label) as HTMLElement; + const isDisabled = (t: HTMLElement) => + t.classList.contains('mat-button-toggle-disabled'); + const box = (id: number) => + el().querySelector( + `[data-testid="dm-admin-${id}"] input` + ) as HTMLInputElement; + const role = (id: number) => + el().querySelector( + `[data-testid="dm-role-${id}"] button[role="switch"]` + ) as HTMLButtonElement; + + function create( + canGrant: boolean, + access = settings(), + tracked: string[] = [] + ) { + api = { + updateAccess: jest.fn(() => of(access)), + access: jest.fn(() => of(access)), + overview: jest.fn(() => of(store.overview)), + baselines: jest.fn(() => + of(tracked.map(id => ({ id, name: id, adopted: true }))) + ), + }; + closeWith = true; + dialog = { open: jest.fn(() => ({ afterClosed: () => of(closeWith) })) }; + TestBed.configureTestingModule({ + imports: [DfDmAccessComponent, NoopAnimationsModule, dmTransloco()], + providers: [ + DataMaskingStore, + { provide: DfDataMaskingApiService, useValue: api }, + { + provide: DfSnackbarService, + useValue: { openSnackBar: jest.fn(), openError: jest.fn() }, + }, + ], + }); + TestBed.overrideProvider(MatDialog, { useValue: dialog }); + store = TestBed.inject(DataMaskingStore); + store.overview = { + mode: 'preview', + scan: { status: 'idle', columns: 3, finishedAt: '2026-10-28T00:00:00Z' }, + counts: { masked: 1, review: 1, clear: 1 }, + you: { canGrant, seesRealValues: false }, + }; + store.access = access; + fixture = TestBed.createComponent(DfDmAccessComponent); + fixture.detectChanges(); + } + + it('the super admin can loosen anything', () => { + create(true); + expect(el().querySelector('[data-testid="dm-access-lock"]')).toBeNull(); + expect(isDisabled(toggle('dm-admin-mode', 'All admins'))).toBe(false); + expect(isDisabled(toggle('dm-ai-mode', 'Same as admins'))).toBe(false); + expect(box(3).disabled).toBe(false); + expect(role(2).disabled).toBe(false); + }); + + it('any other admin can still tighten: the loosening choices lock', () => { + create(false); + const lock = el().querySelector('[data-testid="dm-access-lock"]'); + expect(lock?.textContent?.trim()).toBe( + 'lock Only the super admin can give access.' + ); + // Narrower modes stay open, wider ones lock. + expect(isDisabled(toggle('dm-admin-mode', 'No one'))).toBe(false); + expect(isDisabled(toggle('dm-admin-mode', 'All admins'))).toBe(true); + // AI back to masked is fine; same as admins is not. + expect(isDisabled(toggle('dm-ai-mode', 'Always masked'))).toBe(false); + expect(isDisabled(toggle('dm-ai-mode', 'Same as admins'))).toBe(true); + // Any grant can be removed; nobody new can be granted. + expect(box(2).disabled).toBe(false); + expect(box(3).disabled).toBe(true); + expect(box(1).disabled).toBe(true); + // An enabled role can be disabled; a disabled one cannot be enabled. + expect(role(1).disabled).toBe(false); + expect(role(2).disabled).toBe(true); + }); + + it('removing a grant sends only what is left, with no new durations', () => { + create(false); + box(2).click(); + expect(api.updateAccess).toHaveBeenCalledWith({ admins: { grants: [] } }); + }); + + it('only durations that end sooner than the grant are open to others', () => { + create(false); + const cmp = fixture.componentInstance; + const priya = settings().admins.available[1]; + expect(cmp.durationLocked(priya, '1h')).toBe(false); + expect(cmp.durationLocked(priya, '8h')).toBe(true); + expect(cmp.durationLocked(priya, 'forever')).toBe(true); + // A grant until turned off can be shortened to anything finite. + TestBed.resetTestingModule(); + const forever = settings(); + forever.admins.grants[0].expiresAt = null; + create(false, forever); + expect(fixture.componentInstance.durationLocked(priya, '7d')).toBe(false); + expect(fixture.componentInstance.durationLocked(priya, 'forever')).toBe( + false + ); + }); + + it('scope: narrowing is open to others, widening to every class is not', () => { + create(false); + const cmp = fixture.componentInstance; + const support = settings().roles[0]; + expect(cmp.scopeLocked(support, 'all')).toBe(true); + expect( + cmp.scopeLocked({ ...support, scope: 'all' }, 'except_restricted') + ).toBe(false); + }); + + it('never shows or sends a grant that has expired since the page loaded', () => { + const a = settings(); + a.admins.grants.push({ + adminId: 3, + name: 'Sam Lee', + email: 's@x', + expiresAt: inHours(-1), + }); + expect(liveGrants(a).map(g => g.adminId)).toEqual([2]); + create(true, a); + expect(box(3).checked).toBe(false); + // Granting Kevin re-sends Priya (kept) but not Sam's expired grant, + // which the server would otherwise treat as a fresh 8 hour grant. + box(1).click(); + expect(api.updateAccess).toHaveBeenCalledWith({ + admins: { grants: [{ adminId: 2 }, { adminId: 1, duration: '8h' }] }, + }); + }); + + it('re-reads access every time the tab opens', () => { + create(true); + expect(api.access).toHaveBeenCalledTimes(1); + // A revoke made elsewhere (CLI, another admin) shows on the next visit. + api.access.mockReturnValue( + of(settings({ admins: { ...settings().admins, grants: [] } })) + ); + TestBed.createComponent(DfDmAccessComponent).detectChanges(); + expect(api.access).toHaveBeenCalledTimes(2); + expect(store.access?.admins.grants).toEqual([]); + }); + + it('a role change sends only that role, with the version it was read at', () => { + create(true, { ...settings(), version: 'v7' }); + role(2).click(); + expect(api.updateAccess).toHaveBeenCalledWith({ + roles: [{ roleId: 2, enabled: true, scope: 'except_restricted' }], + version: 'v7', + }); + }); + + it('when a grant ends, the tab re-reads access without a reload', fakeAsync(() => { + const a = settings(); + a.admins.grants[0].expiresAt = new Date(Date.now() + 5000).toISOString(); + create(true, a); + expect(api.access).toHaveBeenCalledTimes(1); + api.access.mockReturnValue( + of(settings({ admins: { ...settings().admins, grants: [] } })) + ); + tick(6500); + fixture.detectChanges(); + expect(api.access).toHaveBeenCalledTimes(2); + expect(box(2).checked).toBe(false); + fixture.destroy(); + })); + + it('All admins asks first; cancel changes nothing', () => { + create(true); + closeWith = false; + fixture.componentInstance.setAdminMode('all'); + expect(dialog.open).toHaveBeenCalledTimes(1); + expect(dialog.open.mock.calls[0][1].data.message).toBe( + 'All admins will see real values in 2 columns.' + ); + expect(api.updateAccess).not.toHaveBeenCalled(); + closeWith = true; + fixture.componentInstance.setAdminMode('all'); + expect(api.updateAccess).toHaveBeenCalledWith({ admins: { mode: 'all' } }); + // Narrowing never asks. + dialog.open.mockClear(); + store.access = { + ...settings(), + admins: { ...settings().admins, mode: 'all' }, + }; + fixture.componentInstance.setAdminMode('none'); + expect(dialog.open).not.toHaveBeenCalled(); + }); + + it('AI "Same as admins" asks first; back to masked never does', () => { + create(true); + fixture.componentInstance.setAi('same_as_admins'); + expect(dialog.open).toHaveBeenCalledTimes(1); + expect(api.updateAccess).toHaveBeenCalledWith({ ai: 'same_as_admins' }); + dialog.open.mockClear(); + store.access = { ...settings(), ai: 'same_as_admins' }; + fixture.componentInstance.setAi('masked'); + expect(dialog.open).not.toHaveBeenCalled(); + }); + + it('marks durations longer than a tracked baseline allows', () => { + create(true, settings(), ['gdpr', 'hipaa']); + const cmp = fixture.componentInstance; + // HIPAA (8 h) is the strictest tracked one. + expect(cmp.exceeds('1h')).toBeNull(); + expect(cmp.exceeds('8h')).toBeNull(); + expect(cmp.exceeds('24h')).toBe('HIPAA'); + expect(cmp.exceeds('forever')).toBe('HIPAA'); + TestBed.resetTestingModule(); + create(true); + expect(fixture.componentInstance.exceeds('forever')).toBeNull(); + }); + + it('the duration list grows to fit "Exceeds HIPAA" on one line; otherwise it matches its select', () => { + const durationSelect = () => + fixture.debugElement + .queryAll(By.directive(MatSelect)) + .map(d => d.componentInstance as MatSelect) + .find(s => s.value === '8h' || s.value === '24h' || s.value === '1h'); + create(true, settings(), ['hipaa']); + expect(durationSelect()!.panelWidth).toBeNull(); + TestBed.resetTestingModule(); + create(true); + expect(durationSelect()!.panelWidth).toBe('auto'); + }); +}); diff --git a/src/app/adf-data-masking/df-dm-access/df-dm-access.component.ts b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.ts new file mode 100644 index 000000000..359bbdc41 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-access/df-dm-access.component.ts @@ -0,0 +1,426 @@ +import { + ChangeDetectorRef, + Component, + DoCheck, + OnDestroy, + OnInit, + ViewChild, + inject, +} from '@angular/core'; +import { NgFor, NgIf } from '@angular/common'; +import { + MatButtonToggleGroup, + MatButtonToggleModule, +} from '@angular/material/button-toggle'; +import { MatDialog, MatDialogModule } from '@angular/material/dialog'; +import { MatCheckboxModule } from '@angular/material/checkbox'; +import { MatIconModule } from '@angular/material/icon'; +import { MatSelectModule } from '@angular/material/select'; +import { MatSlideToggleModule } from '@angular/material/slide-toggle'; +import { TranslocoPipe } from '@ngneat/transloco'; +import { DfSkeletonComponent } from 'src/app/shared/components/df-skeleton/df-skeleton.component'; +import { DfEmptyStateComponent } from 'src/app/shared/components/df-empty-state/df-empty-state.component'; +import { DataMaskingStore } from '../data-masking.store'; +import { baselineDef } from '../data-masking.catalog'; +import { + DfDmConfirmDialogComponent, + DmConfirmData, +} from '../df-data-masking/df-dm-confirm-dialog.component'; +import { + AccessPatch, + AccessSettings, + AdminAccessMode, + AdminRow, + GrantDuration, + RoleAccess, +} from '../data-masking.types'; +import { shortWhen } from '../dm-time'; + +export const DURATIONS: GrantDuration[] = ['1h', '8h', '24h', '7d', 'forever']; +/** + * Grants still in force. The page can outlive a grant; PATCH sends the + * complete list, and a grant sent again after it expired would come back + * as a fresh one, so expired grants are never shown or sent. + */ +export function liveGrants( + a: AccessSettings +): AccessSettings['admins']['grants'] { + const now = Date.now(); + return a.admins.grants.filter( + g => g.expiresAt === null || Date.parse(g.expiresAt) > now + ); +} + +/** Who sees real values, narrowest first (the server's loosening order). */ +const ADMIN_RANK: Record = { + none: 0, + selected: 1, + all: 2, +}; + +const DURATION_MS: Record, number> = { + '1h': 3_600_000, + '8h': 8 * 3_600_000, + '24h': 24 * 3_600_000, + '7d': 7 * 86_400_000, +}; + +/** The smallest preset that still covers what is left on a grant. */ +export function durationFor(expiresAt: string | null): GrantDuration { + if (!expiresAt) return 'forever'; + const left = Date.parse(expiresAt) - Date.now(); + return ( + (Object.keys(DURATION_MS) as (keyof typeof DURATION_MS)[]).find( + d => left <= DURATION_MS[d] + 60_000 + ) ?? '7d' + ); +} + +/** + * Access tab: who sees real values. Admins (none / selected / all, each + * grant timed), AI and MCP sessions, and roles. Each change saves on its + * own. Without grant rights everything is read-only, except removing your + * own grant: narrowing is always allowed. + */ +@Component({ + selector: 'df-dm-access', + standalone: true, + imports: [ + NgIf, + NgFor, + MatButtonToggleModule, + MatCheckboxModule, + MatDialogModule, + MatIconModule, + MatSelectModule, + MatSlideToggleModule, + TranslocoPipe, + DfSkeletonComponent, + DfEmptyStateComponent, + ], + templateUrl: './df-dm-access.component.html', + styleUrls: ['./df-dm-access.component.scss'], +}) +export class DfDmAccessComponent implements OnInit, DoCheck, OnDestroy { + readonly store = inject(DataMaskingStore); + private dialog = inject(MatDialog); + private cdr = inject(ChangeDetectorRef); + readonly durations = DURATIONS; + /** Durations picked this session, so the select shows what was chosen. */ + private picked = new Map(); + private expiryTimer?: ReturnType; + private timedFor: AccessSettings | null = null; + + @ViewChild('adminGroup') adminGroup?: MatButtonToggleGroup; + @ViewChild('aiGroup') aiGroup?: MatButtonToggleGroup; + + ngOnInit(): void { + // Re-read on every visit: access may have changed elsewhere (CLI, + // another admin, a grant that ended) since the page last read it. + this.store.loadAccess(!!this.store.access); + if (!this.store.trackedBaselines) this.store.loadTrackedBaselines(); + } + + ngDoCheck(): void { + if (this.store.access !== this.timedFor) { + this.timedFor = this.store.access; + this.scheduleExpiry(); + } + } + + ngOnDestroy(): void { + clearTimeout(this.expiryTimer); + } + + /** + * When the next grant ends, re-read access (and the overview), so an + * ended grant leaves the screen without a reload. + */ + private scheduleExpiry(): void { + clearTimeout(this.expiryTimer); + const a = this.store.access; + if (!a) return; + const now = Date.now(); + const next = liveGrants(a) + .map(g => (g.expiresAt ? Date.parse(g.expiresAt) : NaN)) + .filter(t => !isNaN(t)) + .sort((x, y) => x - y)[0]; + if (next === undefined) return; + // setTimeout holds at most ~24.8 days; a longer wait re-arms itself. + const wait = Math.min(Math.max(next - now + 1000, 0), 2 ** 31 - 1); + this.expiryTimer = setTimeout(() => { + this.store.loadAccess(true); + this.timedFor = null; + this.cdr.markForCheck(); + }, wait); + } + + /** + * The strictest grant limit among tracked Compliance baselines, as + * { name, hours }, or null when no tracked baseline sets one. + */ + get grantCap(): { name: string; hours: number } | null { + let cap: { name: string; hours: number } | null = null; + for (const id of this.store.trackedBaselines ?? []) { + const b = baselineDef(id); + if (b && (!cap || b.grantMaxHours < cap.hours)) { + cap = { name: b.name, hours: b.grantMaxHours }; + } + } + return cap; + } + + /** The tracked baseline a duration would break, or null. */ + exceeds(d: GrantDuration): string | null { + const cap = this.grantCap; + if (!cap) return null; + const ms = d === 'forever' ? Infinity : DURATION_MS[d]; + return ms > cap.hours * 3_600_000 ? cap.name : null; + } + + /** + * The duration list matches its select's width, unless an option carries + * an "Exceeds …" marker: then it grows to fit the longest option, so no + * option text wraps. + */ + get durationPanelWidth(): 'auto' | null { + return this.durations.some(d => this.exceeds(d)) ? null : 'auto'; + } + + get access(): AccessSettings | null { + return this.store.access; + } + + /** + * The super admin may loosen access. Any admin may tighten it (the + * server's rule): fewer admins, a grant removed or shortened, AI masked, + * a role disabled or narrowed. Only the loosening choices lock. + */ + get canGrant(): boolean { + return !!this.store.overview?.you.canGrant; + } + + isGranted(a: AdminRow): boolean { + return !!this.grantOf(a); + } + + private grantOf(a: AdminRow) { + return this.access + ? liveGrants(this.access).find(g => g.adminId === a.adminId) + : undefined; + } + + /** Granting needs the super admin; any grant can be taken away. */ + checkboxDisabled(a: AdminRow): boolean { + return !this.canGrant && !this.isGranted(a); + } + + /** A wider admin mode than the current one needs the super admin. */ + modeLocked(mode: AdminAccessMode): boolean { + const current = this.access?.admins.mode ?? 'none'; + return !this.canGrant && ADMIN_RANK[mode] > ADMIN_RANK[current]; + } + + /** A duration that would end later than the grant does now. */ + durationLocked(a: AdminRow, d: GrantDuration): boolean { + if (this.canGrant) return false; + const g = this.grantOf(a); + if (!g) return true; + if (d === 'forever') return g.expiresAt !== null; + if (g.expiresAt === null) return false; + return Date.now() + DURATION_MS[d] > Date.parse(g.expiresAt); + } + + /** AI = same as admins needs the super admin; back to masked never does. */ + aiLocked(ai: AccessSettings['ai']): boolean { + return !this.canGrant && ai === 'same_as_admins' && this.access?.ai !== ai; + } + + /** Enabling a role needs the super admin; disabling one never does. */ + roleLocked(r: RoleAccess): boolean { + return !this.canGrant && !r.enabled; + } + + /** Widening a role to every class needs the super admin. */ + scopeLocked(r: RoleAccess, scope: RoleAccess['scope']): boolean { + return !this.canGrant && scope === 'all' && r.scope !== 'all'; + } + + durationOf(a: AdminRow): GrantDuration { + const g = this.grantOf(a); + return this.picked.get(a.adminId) ?? durationFor(g?.expiresAt ?? null); + } + + untilOf(a: AdminRow): string { + const g = this.grantOf(a); + return g?.expiresAt ? shortWhen(g.expiresAt) : ''; + } + + setAdminMode(mode: AdminAccessMode): void { + const a = this.access; + if (!a || a.admins.mode === mode) return; + const go = () => + this.save({ admins: { mode } }, { ...a, admins: { ...a.admins, mode } }); + if (mode !== 'all') { + go(); + return; + } + // Every admin at once is the widest grant there is: say what it means. + this.confirm( + { + title: this.store.t('access.confirmAll.title'), + message: this.store.t('access.confirmAll.message', { + columns: this.store.plural('columns', this.store.maskedTotal), + }), + confirm: this.store.t('access.confirmAll.confirm'), + cancel: this.store.t('cancel'), + danger: true, + }, + go + ); + } + + toggleAdmin(admin: AdminRow, on: boolean): void { + const a = this.access; + if (!a) return; + const keep = liveGrants(a).filter(g => g.adminId !== admin.adminId); + const grants = on + ? [ + ...keep, + { + adminId: admin.adminId, + name: admin.name, + email: admin.email, + expiresAt: new Date(Date.now() + DURATION_MS['8h']).toISOString(), + }, + ] + : keep; + const undo = this.rememberPick(admin.adminId); + if (on) this.picked.set(admin.adminId, '8h'); + else this.picked.delete(admin.adminId); + this.save( + { + admins: { + grants: grants.map(g => ({ + adminId: g.adminId, + ...(g.adminId === admin.adminId && on + ? { duration: '8h' as GrantDuration } + : {}), + })), + }, + }, + { ...a, admins: { ...a.admins, grants } }, + undo + ); + } + + setDuration(admin: AdminRow, duration: GrantDuration): void { + const a = this.access; + if (!a) return; + const undo = this.rememberPick(admin.adminId); + this.picked.set(admin.adminId, duration); + const expiresAt = + duration === 'forever' + ? null + : new Date(Date.now() + DURATION_MS[duration]).toISOString(); + const grants = [ + ...liveGrants(a).filter(g => g.adminId !== admin.adminId), + ...a.admins.grants + .filter(g => g.adminId === admin.adminId) + .map(g => ({ ...g, expiresAt })), + ]; + this.save( + { + admins: { + grants: grants.map(g => ({ + adminId: g.adminId, + ...(g.adminId === admin.adminId ? { duration } : {}), + })), + }, + }, + { ...a, admins: { ...a.admins, grants } }, + undo + ); + } + + setAi(ai: AccessSettings['ai']): void { + const a = this.access; + if (!a || a.ai === ai) return; + const go = () => this.save({ ai }, { ...a, ai }); + if (ai === 'masked') { + go(); + return; + } + this.confirm( + { + title: this.store.t('access.confirmAi.title'), + message: this.store.t('access.confirmAi.message'), + confirm: this.store.t('access.confirmAi.confirm'), + cancel: this.store.t('cancel'), + danger: true, + }, + go + ); + } + + /** Sends only the role that changed; the others stay as the server has them. */ + setRole(role: RoleAccess, patch: Partial): void { + const a = this.access; + if (!a) return; + const changed = { ...role, ...patch }; + const roles = a.roles.map(r => (r.roleId === role.roleId ? changed : r)); + this.save( + { + roles: [ + { + roleId: changed.roleId, + enabled: changed.enabled, + scope: changed.scope, + }, + ], + }, + { ...a, roles } + ); + } + + /** + * Loosening confirms first. A cancel puts the toggles back: a group has + * already moved to the clicked value on its own. + */ + private confirm(data: DmConfirmData, go: () => void): void { + this.dialog + .open(DfDmConfirmDialogComponent, { data, autoFocus: 'dialog' }) + .afterClosed() + .subscribe(ok => { + if (ok) { + go(); + } else { + const a = this.store.access; + if (a && this.adminGroup) this.adminGroup.value = a.admins.mode; + if (a && this.aiGroup) this.aiGroup.value = a.ai; + } + this.cdr.markForCheck(); + }); + } + + /** Restores this admin's picked duration if the save fails. */ + private rememberPick(adminId: number): () => void { + const had = this.picked.has(adminId); + const prev = this.picked.get(adminId); + return () => { + if (had && prev) this.picked.set(adminId, prev); + else this.picked.delete(adminId); + }; + } + + private save( + patch: AccessPatch, + optimistic: AccessSettings, + onError?: () => void + ): void { + this.store.updateAccess(patch, optimistic, onError); + } + + trackAdmin = (_: number, a: AdminRow) => a.adminId; + trackRole = (_: number, r: RoleAccess) => r.roleId; +} diff --git a/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.html b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.html new file mode 100644 index 000000000..b13a23a06 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.html @@ -0,0 +1,71 @@ +
    +
    + +
    + + + + + + + + +
    +

    {{ g.day }}

    +
      +
    1. + + + {{ e.text }} + {{ e.target }} + + {{ e.actor }} + +
      2. +
    +
    + +     + + + +     +
    diff --git a/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.scss b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.scss new file mode 100644 index 000000000..0b71f07f2 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.scss @@ -0,0 +1,152 @@ +:host { + display: block; +} + +.activity { + max-width: 920px; +} + +.filters { + display: flex; + flex-wrap: wrap; + gap: var(--df-space-1); + margin-bottom: var(--df-space-4); +} + +.filter { + background: transparent; + border: 1px solid var(--df-border); + border-radius: 999px; + color: var(--df-text-2); + cursor: pointer; + font: inherit; + font-size: var(--df-font-size-sm); + height: 30px; + padding: 0 var(--df-space-3); + + &:hover { + background: var(--df-hover); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + &.is-on { + background: var(--df-accent-soft); + border-color: transparent; + color: var(--df-accent-strong); + font-weight: var(--df-font-weight-medium); + } +} + +.day + .day { + margin-top: var(--df-space-5); +} + +.day-head { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + margin: 0 0 var(--df-space-1); +} + +.events { + border-top: 1px solid var(--df-border-2); + list-style: none; + margin: 0; + padding: 0; +} + +.event { + align-items: center; + border-bottom: 1px solid var(--df-border-2); + display: grid; + gap: var(--df-space-3); + grid-template-columns: 28px minmax(0, 1fr) minmax(90px, auto) 104px; + min-height: 44px; + padding: var(--df-space-1) var(--df-space-2); +} + +.ico { + align-items: center; + background: var(--df-surface-2); + border-radius: 50%; + color: var(--df-text-muted); + display: inline-flex; + height: 28px; + justify-content: center; + width: 28px; + + mat-icon { + font-size: 16px; + height: 16px; + width: 16px; + } + + .is-blocked & { + background: var(--df-danger-soft); + color: var(--df-danger); + } + + .is-access & { + background: var(--df-accent-soft); + color: var(--df-accent); + } + + .is-mode &, + .is-baseline & { + background: var(--df-accent-soft); + color: var(--df-accent); + } +} + +.text { + color: var(--df-text); + font-size: var(--df-font-size-sm); + line-height: var(--df-lh-base); +} + +.target { + color: var(--df-text-2); + font-family: var(--df-font-mono); + font-size: var(--df-font-size-xs); + margin-left: var(--df-space-1); +} + +.actor { + color: var(--df-text-2); + font-size: var(--df-font-size-sm); + white-space: nowrap; +} + +.when { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + text-align: right; + white-space: nowrap; +} + +@media (max-width: 760px) { + .event { + grid-template-columns: 28px minmax(0, 1fr) auto; + row-gap: 2px; + } + + .actor { + grid-column: 2; + grid-row: 2; + font-size: var(--df-font-size-xs); + } + + .when { + grid-column: 3; + grid-row: 1; + } +} + +.older { + display: block; + margin: var(--df-space-4) auto 0; +} diff --git a/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.spec.ts b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.spec.ts new file mode 100644 index 000000000..194fdc399 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.spec.ts @@ -0,0 +1,98 @@ +import { TestBed } from '@angular/core/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { of } from 'rxjs'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { DataMaskingStore } from '../data-masking.store'; +import { DfDataMaskingApiService } from '../data-masking-api.service'; +import { dmTransloco } from '../dm-testing'; +import { ActivityEvent, ActivityQuery } from '../data-masking.types'; +import { DfDmActivityComponent } from './df-dm-activity.component'; + +/** + * A busy Preview hour: 250 "would mask" rows on top of three older blocked + * attempts. The server filters by kind and pages (ActivityHandler). + */ +function server(): ActivityEvent[] { + const out: ActivityEvent[] = []; + let id = 1; + const t0 = Date.parse('2026-10-29T00:00:00Z'); + for (let i = 0; i < 3; i++) { + out.push({ + id: id++, + at: new Date(t0 + i * 1000).toISOString(), + actor: 'MCP: crm_mcp', + kind: 'blocked', + text: 'Blocked an attempt to change the masking mode', + }); + } + for (let i = 0; i < 250; i++) { + out.push({ + id: id++, + at: new Date(t0 + 60_000 + i * 1000).toISOString(), + actor: 'mobile-app', + kind: 'preview', + text: 'Would mask 3 columns', + target: `sports.t${i}`, + }); + } + return out; +} + +function serve(q: ActivityQuery = {}) { + const all = server() + .filter(e => !q.kinds?.length || q.kinds.includes(e.kind)) + .filter(e => !q.before || e.id < q.before) + .sort((a, b) => b.id - a.id); + const limit = q.limit ?? 200; + return { events: all.slice(0, limit), hasMore: all.length > limit }; +} + +describe('DfDmActivityComponent', () => { + let api: { activity: jest.Mock }; + + function create() { + api = { activity: jest.fn((q: ActivityQuery) => of(serve(q))) }; + TestBed.configureTestingModule({ + imports: [DfDmActivityComponent, NoopAnimationsModule, dmTransloco()], + providers: [ + DataMaskingStore, + { provide: DfDataMaskingApiService, useValue: api }, + { + provide: DfSnackbarService, + useValue: { openSnackBar: jest.fn(), openError: jest.fn() }, + }, + ], + }); + const fixture = TestBed.createComponent(DfDmActivityComponent); + fixture.detectChanges(); + return fixture; + } + + it('the Blocked filter shows the blocked events the server holds', () => { + const fixture = create(); + fixture.componentInstance.setFilter('blocked'); + fixture.detectChanges(); + const blocked = fixture.componentInstance.groups.flatMap(g => g.events); + expect(blocked.length).toBe(3); + expect(api.activity).toHaveBeenLastCalledWith({ + kinds: ['blocked'], + limit: 200, + }); + }); + + it('All pages back with Load older', () => { + const fixture = create(); + const el = fixture.nativeElement as HTMLElement; + const count = () => + fixture.componentInstance.groups.flatMap(g => g.events).length; + expect(count()).toBe(200); + const older = el.querySelector( + '[data-testid="dm-activity-older"]' + ) as HTMLButtonElement; + expect(older).toBeTruthy(); + older.click(); + fixture.detectChanges(); + expect(count()).toBe(253); + expect(el.querySelector('[data-testid="dm-activity-older"]')).toBeNull(); + }); +}); diff --git a/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.ts b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.ts new file mode 100644 index 000000000..a4f6208b9 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-activity/df-dm-activity.component.ts @@ -0,0 +1,152 @@ +import { Component, OnInit, inject } from '@angular/core'; +import { NgClass, NgFor, NgIf } from '@angular/common'; +import { MatButtonModule } from '@angular/material/button'; +import { MatIconModule } from '@angular/material/icon'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { TranslocoPipe } from '@ngneat/transloco'; +import { DfEmptyStateComponent } from 'src/app/shared/components/df-empty-state/df-empty-state.component'; +import { DfSkeletonComponent } from 'src/app/shared/components/df-skeleton/df-skeleton.component'; +import { DataMaskingStore } from '../data-masking.store'; +import { ActivityEvent, ActivityKind } from '../data-masking.types'; +import { relTime } from '../dm-time'; + +export type ActivityFilter = + | 'all' + | 'changes' + | 'access' + | 'scans' + | 'blocked' + | 'preview'; + +const KINDS: Record = { + all: null, + changes: ['mode', 'decision', 'rule', 'baseline'], + access: ['access'], + scans: ['scan'], + blocked: ['blocked'], + preview: ['preview'], +}; + +const ICONS: Record = { + mode: 'toggle_on', + decision: 'how_to_reg', + access: 'key', + scan: 'radar', + blocked: 'block', + rule: 'rule', + baseline: 'verified_user', + preview: 'visibility', +}; + +interface DayGroup { + day: string; + events: ActivityEvent[]; +} + +/** Activity tab: what changed, who did it, when. Newest first, by day. */ +@Component({ + selector: 'df-dm-activity', + standalone: true, + imports: [ + NgIf, + NgFor, + NgClass, + MatButtonModule, + MatIconModule, + MatTooltipModule, + TranslocoPipe, + DfEmptyStateComponent, + DfSkeletonComponent, + ], + templateUrl: './df-dm-activity.component.html', + styleUrls: ['./df-dm-activity.component.scss'], +}) +export class DfDmActivityComponent implements OnInit { + readonly store = inject(DataMaskingStore); + private readonly baseFilters: ActivityFilter[] = [ + 'all', + 'changes', + 'access', + 'scans', + 'blocked', + ]; + filter: ActivityFilter = 'all'; + private sawPreview = false; + + /** + * Preview's "would mask" rows get their own chip while Preview is on, or + * once any have been listed. + */ + get filters(): ActivityFilter[] { + if (!this.sawPreview) { + this.sawPreview = (this.store.activity ?? []).some( + e => e.kind === 'preview' + ); + } + return this.sawPreview || + this.store.mode === 'preview' || + this.filter === 'preview' + ? [...this.baseFilters, 'preview'] + : this.baseFilters; + } + + ngOnInit(): void { + this.store.showActivity(KINDS[this.filter]); + } + + /** Each chip asks the server for its kinds, so none hides behind another. */ + setFilter(f: ActivityFilter): void { + if (f === this.filter) return; + this.filter = f; + this.store.showActivity(KINDS[f]); + } + + get groups(): DayGroup[] { + const kinds = KINDS[this.filter]; + // The server filters; this only guards a list that is still loading. + const events = (this.store.activity ?? []) + .filter(e => !kinds || kinds.includes(e.kind)) + .sort((a, b) => Date.parse(b.at) - Date.parse(a.at) || b.id - a.id); + const out: DayGroup[] = []; + for (const e of events) { + const day = this.dayLabel(e.at); + const last = out[out.length - 1]; + if (last && last.day === day) last.events.push(e); + else out.push({ day, events: [e] }); + } + return out; + } + + icon(e: ActivityEvent): string { + return ICONS[e.kind] ?? 'history'; + } + + ago(e: ActivityEvent): string { + return relTime(e.at); + } + + exact(e: ActivityEvent): string { + return new Date(e.at).toLocaleString('en'); + } + + private dayLabel(iso: string): string { + const d = new Date(iso); + const today = new Date(); + const y = new Date(); + y.setDate(today.getDate() - 1); + if (d.toDateString() === today.toDateString()) { + return this.store.t('activity.today'); + } + if (d.toDateString() === y.toDateString()) { + return this.store.t('activity.yesterday'); + } + return d.toLocaleDateString('en', { + weekday: 'long', + month: 'short', + day: 'numeric', + }); + } + + trackDay = (_: number, g: DayGroup) => g.day; + trackEvent = (_: number, e: ActivityEvent) => e.id; +} diff --git a/src/app/adf-data-masking/df-dm-columns/_dm-columns-first-run.scss b/src/app/adf-data-masking/df-dm-columns/_dm-columns-first-run.scss new file mode 100644 index 000000000..3b8858d8f --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/_dm-columns-first-run.scss @@ -0,0 +1,165 @@ +// Columns tab first run: the never-scanned call to action with one +// shortcut per compliance framework, and the first scan's progress. +@mixin dm-columns-first-run { + .first-run { + display: flex; + flex-direction: column; + align-items: center; + padding-bottom: var(--df-space-8); + } + + // The first scan could not read a database: one line, under the action. + .first-run-failed { + align-items: center; + color: var(--df-danger); + display: flex; + font-size: var(--df-font-size-sm); + gap: var(--df-space-1); + margin: calc(-1 * var(--df-space-5)) 0 var(--df-space-6); + + mat-icon { + font-size: 16px; + height: 16px; + width: 16px; + } + } + + // Sits under the empty state's button, tucked into its bottom padding. + .baselines { + align-items: center; + display: flex; + flex-direction: column; + gap: var(--df-space-2); + margin-top: calc(-1 * var(--df-space-5)); + max-width: 100%; + } + + .baselines-label { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + } + + .baselines-list { + display: flex; + flex-wrap: wrap; + gap: var(--df-space-2); + justify-content: center; + list-style: none; + margin: 0; + padding: 0 var(--df-space-4); + } + + .baseline { + align-items: center; + background: var(--df-surface); + border: 1px solid var(--df-border); + border-radius: 999px; + color: var(--df-text-2); + cursor: pointer; + display: inline-flex; + font: inherit; + font-size: var(--df-font-size-sm); + gap: 6px; + height: 30px; + padding: 0 var(--df-space-3) 0 var(--df-space-2); + transition: + background-color var(--df-duration-fast) var(--df-ease-standard), + border-color var(--df-duration-fast) var(--df-ease-standard); + + mat-icon { + color: var(--df-text-muted); + font-size: 16px; + height: 16px; + width: 16px; + } + + &:hover { + background: var(--df-hover); + border-color: var(--df-text-faint); + color: var(--df-text); + + mat-icon { + color: var(--df-accent); + } + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + } + + // First scan: the icon chip, a live label and a thin bar. + .first-scan { + align-items: center; + display: flex; + flex-direction: column; + gap: var(--df-space-3); + padding: var(--df-space-8) var(--df-space-5); + text-align: center; + } + + .first-scan-icon { + align-items: center; + background: var(--df-accent-soft); + border-radius: var(--df-radius); + color: var(--df-accent); + display: inline-flex; + height: var(--df-space-8); + justify-content: center; + width: var(--df-space-8); + + mat-icon { + font-size: var(--df-font-size-2xl); + height: var(--df-font-size-2xl); + width: var(--df-font-size-2xl); + } + } + + .first-scan-title { + color: var(--df-text); + font-size: var(--df-font-size-lg); + font-weight: var(--df-font-weight-heading); + margin: 0; + } + + .first-scan-bar { + background: var(--df-surface-2); + border: 1px solid var(--df-border-2); + border-radius: 999px; + height: 6px; + overflow: hidden; + width: min(280px, 100%); + + span { + background: var(--df-accent); + border-radius: inherit; + display: block; + height: 100%; + transition: width var(--df-duration-standard) var(--df-ease-standard); + } + + // No percentage yet: a short segment sweeps across. + &.is-indeterminate span { + animation: dm-sweep 1.4s var(--df-ease-standard) infinite; + width: 30% !important; + } + } + + @keyframes dm-sweep { + 0% { + transform: translateX(-100%); + } + 100% { + transform: translateX(340%); + } + } + + @media (prefers-reduced-motion: reduce) { + .first-scan-bar.is-indeterminate span { + animation: none; + width: 100% !important; + opacity: 0.4; + } + } +} diff --git a/src/app/adf-data-masking/df-dm-columns/_dm-columns-rail.scss b/src/app/adf-data-masking/df-dm-columns/_dm-columns-rail.scss new file mode 100644 index 000000000..01a6ef53d --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/_dm-columns-rail.scss @@ -0,0 +1,217 @@ +// Columns tab rail: services, table rows, sensitivity strip slot and the +// to-review badge. +@mixin dm-columns-rail { + .rail { + border-right: 1px solid var(--df-border-2); + max-height: calc(100vh - 260px); + min-height: 240px; + overflow-y: auto; + padding-right: var(--df-space-3); + position: sticky; + scrollbar-width: thin; + top: 0; + } + + .rail-group + .rail-group { + margin-top: var(--df-space-4); + } + + .rail-service { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + margin: 0 0 var(--df-space-1); + padding: 0 var(--df-space-2); + } + + .rail-row { + align-items: center; + background: transparent; + border: 0; + border-radius: var(--df-radius-sm); + // Rows are buttons, but the coverage rows are a div / li: same box. + box-sizing: border-box; + color: var(--df-text-2); + cursor: pointer; + display: grid; + font: inherit; + font-size: var(--df-font-size-sm); + gap: var(--df-space-2); + // Fixed strip slot: names truncate evenly and strips line up. + grid-template-columns: minmax(0, 1fr) var(--dm-strip-slot, 72px) 22px; + height: 32px; + padding: 0 var(--df-space-2); + text-align: left; + width: 100%; + + &:hover { + background: var(--df-hover); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + &.is-selected { + background: var(--df-accent-soft); + color: var(--df-text); + font-weight: var(--df-font-weight-medium); + } + } + + .rail-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .rail-badge { + display: inline-flex; + justify-content: flex-end; + } + + .badge { + background: var(--df-accent-soft); + border-radius: 999px; + // Pulled toward the text colour: 4.5:1 on the soft wash in both themes. + color: color-mix(in srgb, var(--df-accent-strong) 65%, var(--df-text)); + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + line-height: 18px; + min-width: 18px; + padding: 0 5px; + text-align: center; + } + + .rail-check { + color: var(--df-success); + font-size: 16px; + height: 16px; + width: 16px; + } + + // A covered service with no scan yet: its name and its own Scan. + .rail-row.is-pending { + color: var(--df-text-2); + cursor: default; + grid-template-columns: minmax(0, 1fr) auto; + + &:hover { + background: transparent; + } + } + + .rail-scan { + align-items: center; + background: transparent; + border: 1px solid var(--df-border); + border-radius: var(--df-radius-sm); + color: var(--df-accent-strong); + cursor: pointer; + display: inline-flex; + font: inherit; + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + gap: var(--df-space-1); + height: 24px; + padding: 0 var(--df-space-2); + + mat-icon { + font-size: 16px; + height: 16px; + width: 16px; + } + + &:hover:not(:disabled) { + background: var(--df-accent-soft); + border-color: transparent; + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + &:disabled { + color: var(--df-text-faint); + cursor: default; + } + } + + // The scan history could not be read: one muted line where the + // never-scanned group would be. + .rail-unknown { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + margin: 0; + padding: 0 var(--df-space-2); + } + + // A database the last scan could not read: its name (reason on hover and + // for screen readers) and a Retry. + .rail-group.is-failed .rail-service { + color: var(--df-danger); + } + + // Scanned, nothing under this filter: listed quietly so it never looks + // unscanned. A database with tables opens under All. + .rail-row.is-quiet { + color: var(--df-text-muted); + grid-template-columns: minmax(0, 1fr) auto; + + &:is(div) { + cursor: default; + + &:hover { + background: transparent; + } + } + } + + .rail-note { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + white-space: nowrap; + } + + // The rail foot alone, under an empty filter. + .rail.rail-solo { + border-right: 0; + margin: var(--df-space-4) auto 0; + max-height: none; + max-width: 360px; + min-height: 0; + padding-right: 0; + position: static; + } + + // Databases Data Masking cannot mask: listed for the record, muted and + // not selectable. The tooltip (and its sr-only copy) says what that means. + .rail-group.is-uncovered { + border-top: 1px dashed var(--df-border-2); + padding-top: var(--df-space-3); + } + + .rail-list { + list-style: none; + margin: 0; + padding: 0; + } + + .rail-row.is-uncovered { + color: var(--df-text-muted); + cursor: default; + grid-template-columns: minmax(0, 1fr) auto; + + &:hover { + background: transparent; + } + } + + .rail-type { + color: var(--df-text-faint); + font-family: var(--df-font-mono); + font-size: var(--df-font-size-xs); + } +} diff --git a/src/app/adf-data-masking/df-dm-columns/_dm-columns-toolbar.scss b/src/app/adf-data-masking/df-dm-columns/_dm-columns-toolbar.scss new file mode 100644 index 000000000..010eb2d30 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/_dm-columns-toolbar.scss @@ -0,0 +1,184 @@ +// Columns tab toolbar: search, filter chips, Scan now (with its progress +// fill). Included at its place in df-dm-columns.component.scss so the +// cascade order is unchanged. +@mixin dm-columns-toolbar { + .toolbar { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: var(--df-space-3); + margin-bottom: var(--df-space-4); + } + + .search { + align-items: center; + background: var(--df-surface); + border: 1px solid var(--df-border); + border-radius: var(--df-radius-sm); + color: var(--df-text-muted); + display: flex; + gap: var(--df-space-1); + height: 32px; + padding: 0 var(--df-space-2); + width: 248px; + + // One focus ring, on the field; the input's own ring is switched off. + &:focus-within { + border-color: var(--df-accent); + box-shadow: var(--df-focus-ring); + } + + mat-icon { + font-size: 18px; + height: 18px; + width: 18px; + } + + input { + background: transparent; + border: 0; + color: var(--df-text); + flex: 1; + font: inherit; + font-size: var(--df-font-size-sm); + min-width: 0; + outline: none; + + &:focus, + &:focus-visible { + box-shadow: none; + outline: none; + } + + &::placeholder { + color: var(--df-text-faint); + } + + &::-webkit-search-cancel-button { + display: none; + } + } + } + + .search-clear { + background: none; + border: 0; + border-radius: 50%; + color: var(--df-text-muted); + cursor: pointer; + display: inline-flex; + padding: 2px; + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + } + + .filters { + display: flex; + flex-wrap: wrap; + gap: var(--df-space-1); + } + + .filter { + align-items: center; + background: transparent; + border: 1px solid var(--df-border); + border-radius: 999px; + color: var(--df-text-2); + cursor: pointer; + display: inline-flex; + font: inherit; + font-size: var(--df-font-size-sm); + gap: 6px; + height: 30px; + padding: 0 var(--df-space-3); + transition: + background-color var(--df-duration-fast) var(--df-ease-standard), + border-color var(--df-duration-fast) var(--df-ease-standard); + + &:hover { + background: var(--df-hover); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + &.is-on { + background: var(--df-accent-soft); + border-color: transparent; + color: var(--df-accent-strong); + font-weight: var(--df-font-weight-medium); + } + + button { + background: none; + border: 0; + color: inherit; + cursor: pointer; + display: inline-flex; + padding: 0; + } + + mat-icon { + font-size: 16px; + height: 16px; + width: 16px; + } + + code { + font-family: var(--df-font-mono); + font-size: var(--df-font-size-xs); + } + } + + .filter-n { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + + .is-on & { + color: inherit; + } + + .is-review:not(.is-on) & { + color: var(--df-accent-strong); + font-weight: var(--df-font-weight-medium); + } + } + + .scan { + margin-left: auto; + overflow: hidden; + position: relative; + + mat-icon { + font-size: 18px; + height: 18px; + width: 18px; + } + + // Progress fills the button from the left; the label stays readable on + // top. Width changes are not animated, so reduced motion is respected. + &.is-scanning { + cursor: progress; + + &::before { + background: var(--df-accent-soft); + bottom: 0; + content: ''; + left: 0; + position: absolute; + top: 0; + width: var(--dm-scan-progress, 0%); + } + + mat-icon, + .scan-label { + position: relative; + } + } + } +} diff --git a/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.html b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.html new file mode 100644 index 000000000..cdd19ebb6 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.html @@ -0,0 +1,592 @@ + + + + + + +
    + + +
    + + {{ f.label }} + + + + + +
    + + + +
    + + + + + {{ 'dataMasking.columns.table' | transloco }} + + + + {{ r.table }} + + ({{ n }}) + + + + {{ u.label }} · {{ u.type }} + + + +
    + + +
    + +
    + + +
    +
    +

    {{ selectedTable }}

    + + + + + + {{ 'dataMasking.columns.allReviewed' | transloco }} + + + +
    + +
    +
    + {{ + 'dataMasking.columns.column' | transloco + }} + {{ + 'dataMasking.columns.class' | transloco + }} + {{ + 'dataMasking.columns.maskedAs' | transloco + }} + {{ + 'dataMasking.columns.decision' | transloco + }} + {{ + 'dataMasking.columns.evidence' | transloco + }} +
    + + + +
    + + {{ c.column }} + + + + + + + + {{ 'dataMasking.formats.redact' | transloco }} + + + + + + + + + + + + + person + + + + + +
    +
    +
    + +
    +
    +     +
    +
    +
    +     +     + + + +
    +

    + {{ 'dataMasking.columns.failed' | transloco }} +

    +
    + {{ s.label }} + {{ store.scanErrorOf(s) }} + +
    +
    + + +

    + {{ 'dataMasking.columns.scanUnknown' | transloco }} +

    + + + +
    +

    + {{ 'dataMasking.columns.notScanned' | transloco }} +

    +
    + {{ s.label }} + +
    +
    +     + + +
    + + + +
    + {{ r.label }} + {{ + 'dataMasking.columns.noTables' | transloco + }} +
    +     +     +
    + + +
    +

    + {{ 'dataMasking.columns.uncovered' | transloco }} +

    +
      +
    • + {{ u.label }} + {{ u.type }} + {{ + 'dataMasking.columns.uncoveredTip' | transloco + }} +
      • +
    +
    +     + + + + + + + + + + + + + + + +
    + +

    {{ scanningLabel }}

    +
    + +
    +
    + +
    + + + + + + + + + +

    + + {{ 'dataMasking.toast.scanFailed' | transloco: { label: names } }} +

    + +
    +     +     diff --git a/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.scss b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.scss new file mode 100644 index 000000000..43205e04d --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.scss @@ -0,0 +1,458 @@ +@use './dm-columns-toolbar' as *; +@use './dm-columns-rail' as *; +@use './dm-columns-first-run' as *; + +:host { + display: block; +} + +.sr-only { + clip: rect(0 0 0 0); + height: 1px; + overflow: hidden; + position: absolute; + white-space: nowrap; + width: 1px; +} + +// ---------------------------------------------------------------- toolbar + +@include dm-columns-toolbar; + +// -------------------------------------------------------------- first run + +@include dm-columns-first-run; + +// ------------------------------------------------------------------ panes + +.table-select { + display: none; + margin-bottom: var(--df-space-3); + width: 100%; +} + +// Narrow screens: never-scanned services get their Scan buttons here, +// since the rail (where they normally sit) collapses to the select. +.pending-narrow { + display: none; + flex-wrap: wrap; + gap: var(--df-space-2); + margin-bottom: var(--df-space-3); +} + +.panes { + align-items: start; + display: grid; + gap: var(--df-space-5); + grid-template-columns: 272px minmax(0, 1fr); +} + +@include dm-columns-rail; + +// ------------------------------------------------------------------- list + +.list { + min-width: 0; +} + +.list-head { + align-items: center; + display: flex; + gap: var(--df-space-3); + justify-content: space-between; + min-height: 36px; + margin-bottom: var(--df-space-2); +} + +.list-title { + color: var(--df-text); + font-family: var(--df-font-mono); + font-size: var(--df-font-size-lg); + font-weight: var(--df-font-weight-heading); + margin: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.all-done { + align-items: center; + color: var(--df-success); + display: inline-flex; + font-size: var(--df-font-size-sm); + gap: var(--df-space-1); + + mat-icon { + font-size: 18px; + height: 18px; + width: 18px; + } +} + +.grid { + border-top: 1px solid var(--df-border-2); +} + +.row { + align-items: center; + border-bottom: 1px solid var(--df-border-2); + column-gap: var(--df-space-3); + display: grid; + // Every row is its own grid, so the chip and value tracks are fixed + // widths: content-sized tracks would differ row to row. 216px fits the + // longest class chip ("Health & special category" with its lock). + grid-template-columns: + minmax(120px, 1.1fr) 216px minmax(100px, 0.8fr) + 200px 28px; + min-height: var(--df-row-height); + padding: 0 var(--df-space-2); + position: relative; + // Off-screen rows of a wide table skip layout and paint. + content-visibility: auto; + contain-intrinsic-size: auto var(--df-row-height); + transition: background-color var(--df-duration-standard) + var(--df-ease-standard); + + &:not(.head):hover { + background: var(--df-hover); + } + + &:focus-visible { + box-shadow: inset 0 0 0 2px var(--df-accent); + outline: none; + } + + &.head { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + min-height: 32px; + } + + &.is-open { + background: var(--df-surface-2); + border-bottom-color: transparent; + } + + // Undecided rows carry a thin accent edge, not a loud fill. The accent + // means "needs you" here; tier colours only ever mean the kind of data. + &.is-review::before { + background: var(--df-accent); + border-radius: 2px; + bottom: 8px; + content: ''; + left: 0; + position: absolute; + top: 8px; + width: 2px; + } + + // A decision lands: brief wash that settles back. No size change. + &.is-settled { + animation: dm-settle 900ms var(--df-ease-standard); + } +} + +@keyframes dm-settle { + 0% { + background-color: var(--df-accent-soft); + } + 100% { + background-color: transparent; + } +} + +@media (prefers-reduced-motion: reduce) { + .row.is-settled { + animation: none; + } +} + +.c-name { + min-width: 0; + + code { + color: var(--df-text); + display: block; + font-family: var(--df-font-mono); + font-size: var(--df-font-size-xs); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + + .is-clear & { + color: var(--df-text-2); + } + } +} + +.c-class, +.c-value { + display: flex; + min-width: 0; +} + +.dash { + color: var(--df-text-faint); +} + +// "Redact" for a masked column with no class: the format, named. +.c-format { + color: var(--df-text-2); + font-size: var(--df-font-size-sm); +} + +.c-act { + align-items: center; + display: flex; + gap: var(--df-space-1); + justify-content: flex-end; +} + +.act { + background: transparent; + border: 1px solid var(--df-border); + border-radius: var(--df-radius-sm); + color: var(--df-text-2); + cursor: pointer; + font: inherit; + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + height: 26px; + padding: 0 var(--df-space-2); + white-space: nowrap; + + &:hover { + background: var(--df-hover); + color: var(--df-text); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } +} + +.act-keep { + background: var(--df-accent-soft); + border-color: transparent; + color: var(--df-accent-strong); + + &:hover { + background: var(--df-accent-soft); + border-color: var(--df-accent); + color: var(--df-accent-strong); + } +} + +.decided { + border-radius: 50%; + color: var(--df-text-muted); + cursor: default; + font-size: 18px; + height: 18px; + width: 18px; + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } +} + +.exp { + align-items: center; + background: transparent; + border: 0; + border-radius: var(--df-radius-sm); + color: var(--df-text-muted); + cursor: pointer; + display: inline-flex; + height: 28px; + justify-content: center; + padding: 0; + width: 28px; + + &:hover { + background: var(--df-hover); + color: var(--df-text); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + mat-icon { + transition: transform var(--df-duration-standard) var(--df-ease-standard); + } + + &[aria-expanded='true'] mat-icon { + transform: rotate(180deg); + } +} + +.ledger-row { + background: var(--df-surface-2); + border-bottom: 1px solid var(--df-border-2); +} + +// The rest of the table under the review queue: present, but quiet. A +// heavier rule above the first one marks where the queue ends. +.row.is-context-start { + border-top: 2px solid var(--df-border); +} + +.row.is-context { + opacity: 0.75; + transition: opacity var(--df-duration-fast) var(--df-ease-standard); + + &:hover, + &:focus-within, + &.is-open { + opacity: 1; + } +} + +// ---------------------------------------------------------------- narrow + +@media (max-width: 1240px) { + .panes { + gap: var(--df-space-4); + grid-template-columns: 224px minmax(0, 1fr); + } + + .rail { + --dm-strip-slot: 48px; + } +} + +@media (max-width: 760px) { + // Search and an icon-only Scan share the first line; the filter chips + // are one row that scrolls sideways. + .toolbar { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + row-gap: var(--df-space-2); + } + + .search { + width: auto; + } + + .scan { + grid-column: 2; + grid-row: 1; + margin-left: 0; + min-width: 0; + padding: 0 var(--df-space-2); + + .scan-label { + display: none; + } + + mat-icon { + margin: 0; + } + } + + // Chips wrap: every filter stays in view on a phone. + .filters { + grid-column: 1 / -1; + } + + .filter { + flex: 0 0 auto; + } + + // The rail foot (failed / quiet / uncovered databases) under the list. + .rail.rail-solo { + display: block; + } + + .table-select { + display: block; + } + + .pending-narrow { + display: flex; + } + + .panes { + display: block; + } + + .rail:not(.rail-solo), + .list-title { + display: none; + } + + .list-head:not(:has(button, .all-done)) { + display: none; + } +} + +// The list reflows on its own width, not the viewport's, so it holds up +// beside the rail at 1024 as well as alone on a phone. +.list { + container-type: inline-size; +} + +@container (max-width: 700px) { + .row { + grid-template-columns: minmax(0, 1fr) 172px minmax(72px, 112px) 28px; + } + + .row.head > :nth-child(4) { + display: none; + } + + // Decisions drop to their own line under the row; the row stays a row. + .c-act { + grid-column: 1 / -1; + grid-row: 2; + justify-content: flex-start; + padding-bottom: var(--df-space-2); + + &:not(:has(*)) { + display: none; + } + } +} + +@container (max-width: 380px) { + .row.head { + display: none; + } + + // Name on top; class chip and value wrap underneath; decisions last. + .row { + grid-template-areas: + 'name name exp' + 'cls val val' + 'act act act'; + grid-template-columns: max-content minmax(0, 1fr) 28px; + padding: var(--df-space-2); + row-gap: 6px; + } + + .c-name { + grid-area: name; + } + + .c-class { + grid-area: cls; + } + + .c-value { + grid-area: val; + } + + .c-act { + grid-area: act; + padding-bottom: 0; + } + + .c-exp { + grid-area: exp; + } +} diff --git a/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.spec.ts b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.spec.ts new file mode 100644 index 000000000..778e5a85f --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.spec.ts @@ -0,0 +1,367 @@ +import { + ComponentFixture, + TestBed, + fakeAsync, + flush, + tick, +} from '@angular/core/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { provideRouter } from '@angular/router'; +import { of, throwError } from 'rxjs'; +import { DfDmColumnsComponent, FIRST_ROWS } from './df-dm-columns.component'; +import { DmUndoToastService } from '../df-dm-undo-toast/df-dm-undo-toast.component'; +import { DataMaskingStore, summarize } from '../data-masking.store'; +import { DfDataMaskingApiService } from '../data-masking-api.service'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { column, dmTransloco } from '../dm-testing'; +import { MaskedColumn } from '../data-masking.types'; + +describe('DfDmColumnsComponent (review flow)', () => { + let fixture: ComponentFixture; + let store: DataMaskingStore; + let api: { decide: jest.Mock }; + + const cols: MaskedColumn[] = [ + column({ + id: 'crm.a.dob', + table: 'a', + column: 'dob', + verdict: 'review', + class: 'date_of_birth', + tier: 'identity', + example: '1984-••', + format: 'year', + }), + column({ id: 'crm.a.ssn', table: 'a', column: 'ssn' }), + column({ + id: 'crm.a.note', + table: 'a', + column: 'note', + verdict: 'clear', + class: null, + tier: null, + example: null, + }), + column({ + id: 'crm.b.name', + table: 'b', + column: 'name', + verdict: 'review', + class: 'person_name', + tier: 'identity', + example: 'NAME_0001c3d4', + format: 'token', + }), + ]; + + const el = () => fixture.nativeElement as HTMLElement; + const rows = () => + Array.from(el().querySelectorAll('[data-testid="dm-row"]')); + + beforeEach(() => { + api = { + decide: jest.fn((id: string, cls: string | null) => + of({ + ...cols.find(c => c.id === id)!, + class: cls, + verdict: cls ? 'masked' : 'clear', + source: 'admin', + }) + ), + }; + TestBed.configureTestingModule({ + imports: [DfDmColumnsComponent, NoopAnimationsModule, dmTransloco()], + providers: [ + provideRouter([]), + DataMaskingStore, + { provide: DfDataMaskingApiService, useValue: api }, + { + provide: DfSnackbarService, + useValue: { openSnackBar: jest.fn(), openError: jest.fn() }, + }, + { provide: DmUndoToastService, useValue: { open: jest.fn() } }, + ], + }); + store = TestBed.inject(DataMaskingStore); + store.loading = false; + store.overview = { + mode: 'preview', + scan: { status: 'idle', columns: 4, finishedAt: '2026-10-28T00:00:00Z' }, + counts: { masked: 1, review: 2, clear: 1 }, + you: { canGrant: true, seesRealValues: false }, + }; + const list = cols.map(c => ({ ...c })); + store.columns.set('crm', list); + store.services = [ + { + service: 'crm', + label: 'CRM database', + tables: [ + summarize( + 'a', + list.filter(c => c.table === 'a') + ), + summarize( + 'b', + list.filter(c => c.table === 'b') + ), + ], + }, + ]; + fixture = TestBed.createComponent(DfDmColumnsComponent); + fixture.detectChanges(); + }); + + it('opens on the review queue, with the rest of the table after it', () => { + expect(fixture.componentInstance.filter).toBe('review'); + expect(rows().length).toBe(3); + expect(rows()[0].textContent).toContain('dob'); + expect(rows()[1].classList).toContain('is-context'); + expect(el().querySelectorAll('[data-testid="dm-keep"]').length).toBe(1); + }); + + it('keep masked records the decision without removing the row', fakeAsync(() => { + ( + el().querySelector('[data-testid="dm-keep"]') as HTMLButtonElement + ).click(); + fixture.detectChanges(); + expect(api.decide).toHaveBeenCalledWith('crm.a.dob', 'date_of_birth'); + expect(rows().length).toBe(3); + expect(rows()[0].classList).toContain('is-settled'); + expect(rows()[0].querySelector('[data-testid="dm-decided"]')).toBeTruthy(); + expect(store.counts.review).toBe(1); + expect(store.counts.masked).toBe(2); + tick(1000); + })); + + it('offers the next table once this one is reviewed', fakeAsync(() => { + ( + el().querySelector('[data-testid="dm-clear"]') as HTMLButtonElement + ).click(); + fixture.detectChanges(); + expect(api.decide).toHaveBeenCalledWith('crm.a.dob', null); + const next = el().querySelector( + '[data-testid="dm-next-table"]' + ) as HTMLButtonElement; + expect(next).toBeTruthy(); + next.click(); + tick(); + fixture.detectChanges(); + expect(fixture.componentInstance.selectedTable).toBe('b'); + expect(rows()[0].textContent).toContain('name'); + tick(1000); + })); + + it('search narrows to matching columns and says when nothing matches', () => { + const cmp = fixture.componentInstance; + cmp.setFilter('all'); + cmp.search = 'ssn'; + cmp.onSearch(); + fixture.detectChanges(); + expect(rows().map(r => r.textContent)).toEqual([ + expect.stringContaining('ssn'), + ]); + cmp.search = 'zzz'; + cmp.onSearch(); + fixture.detectChanges(); + expect(el().textContent).toContain('No columns match "zzz"'); + }); + + it('rolls back and toasts when the decision fails', () => { + const snack = TestBed.inject(DfSnackbarService) as unknown as { + openError: jest.Mock; + }; + api.decide.mockReturnValueOnce( + throwError(() => ({ status: 500, message: 'boom', kind: 'server' })) + ); + ( + el().querySelector('[data-testid="dm-keep"]') as HTMLButtonElement + ).click(); + fixture.detectChanges(); + expect(store.counts.review).toBe(2); + expect(snack.openError).toHaveBeenCalled(); + }); + it('filters are disjoint, like their counts, and All shows its total', () => { + const cmp = fixture.componentInstance; + cmp.setFilter('masked'); + fixture.detectChanges(); + expect(cmp.count('masked')).toBe(1); + expect(rows().map(r => r.textContent)).toEqual([ + expect.stringContaining('ssn'), + ]); + const chips = Array.from(el().querySelectorAll('.filter')).map(b => + b.textContent?.replace(/\s+/g, ' ').trim() + ); + expect(chips).toEqual([ + 'To review 2', + 'Masked 1', + 'Not sensitive 1', + 'All 4', + ]); + }); + + it('hides To review once nothing is left to review', () => { + const cmp = fixture.componentInstance; + store.overview = { + ...store.overview!, + counts: { masked: 3, review: 0, clear: 1 }, + }; + cmp.setFilter('masked'); + fixture.detectChanges(); + expect(el().querySelector('[data-testid="dm-filter-review"]')).toBeNull(); + }); + + it('keeps focus in the row after "Keep masked" or "Not sensitive"', () => { + const keep = el().querySelector('[data-testid="dm-keep"]') as HTMLElement; + keep.focus(); + keep.click(); + fixture.detectChanges(); + expect(document.activeElement).toBe(rows()[0]); + // The row shortcuts keep working from there. + rows()[0].dispatchEvent(new KeyboardEvent('keydown', { key: 'j' })); + expect(document.activeElement).toBe(rows()[1]); + }); + + it('the grid is one tab stop: only the active row and its controls are tabbable', () => { + const tabbable = (r: Element) => + Array.from(r.querySelectorAll('button, [tabindex]')).filter( + e => (e as HTMLElement).tabIndex >= 0 + ).length; + expect(rows().map(r => (r as HTMLElement).tabIndex)).toEqual([0, -1, -1]); + expect(tabbable(rows()[1])).toBe(0); + expect(tabbable(rows()[0])).toBeGreaterThan(0); + rows()[0].dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown' })); + fixture.detectChanges(); + expect(rows().map(r => (r as HTMLElement).tabIndex)).toEqual([-1, 0, -1]); + }); + + it('the rail is one tab stop: arrows move between tables and open them', () => { + const cmp = fixture.componentInstance; + cmp.setFilter('all'); + fixture.detectChanges(); + const rail = () => + Array.from( + el().querySelectorAll('[data-testid="dm-rail-row"]') + ) as HTMLElement[]; + expect(rail().map(r => r.tabIndex)).toEqual([0, -1]); + rail()[0].dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown' })); + fixture.detectChanges(); + expect(cmp.selectedTable).toBe('b'); + expect(rail().map(r => r.tabIndex)).toEqual([-1, 0]); + expect(document.activeElement).toBe(rail()[1]); + }); + + it('has no Keep masked for a review column without a class', () => { + const list = store.columns.get('crm')!; + list[0] = { ...list[0], class: null, tier: null }; + store.version++; + fixture.detectChanges(); + expect(el().querySelector('[data-testid="dm-keep"]')).toBeNull(); + // The m shortcut must not turn it into "not sensitive" either. + const row = rows()[0] as HTMLElement; + row.dispatchEvent(new KeyboardEvent('keydown', { key: 'm' })); + expect(api.decide).not.toHaveBeenCalled(); + }); + + it('a computed field stored with no class reads "Redact" under Masked as', () => { + // What /column returns for a computed field whose expression can't be + // checked: review, class null, so no format and no example. + const list = store.columns.get('crm')!; + list[0] = { + ...list[0], + column: 'contact_line', + class: null, + tier: null, + format: null, + example: null, + evidence: { + ...list[0].evidence, + lines: [ + { + points: 40, + kind: 'context', + text: 'Built from an expression that could not be checked', + }, + ], + }, + }; + store.version++; + fixture.detectChanges(); + const row = rows()[0] as HTMLElement; + const value = row.querySelector('.c-value')!; + expect(value.textContent?.trim()).toBe('Redact'); + expect(value.querySelector('df-masked-value')).toBeNull(); + expect(row.textContent).not.toMatch(/\bnull\b/i); + // The chip asks for a class instead of claiming "Not sensitive". + expect( + row.querySelector('[data-testid="dm-class-chip"]')!.textContent + ).toContain('Choose a class'); + // A column with a class still draws its masked example. + expect( + (rows()[1] as HTMLElement).querySelector('.c-value df-masked-value') + ).toBeTruthy(); + }); + + it('marks where the review queue ends without an empty spacer row', () => { + expect(el().querySelector('.context-gap')).toBeNull(); + expect(rows()[1].classList).toContain('is-context-start'); + }); + it('gives the table select a spoken name', () => { + const sel = el().querySelector('.table-select mat-select'); + expect(sel?.getAttribute('aria-label')).toBe('Table'); + }); +}); + +describe('DfDmColumnsComponent (wide tables)', () => { + it('opens a 300-column table with the first rows at once, the rest right after', fakeAsync(() => { + TestBed.configureTestingModule({ + imports: [DfDmColumnsComponent, NoopAnimationsModule, dmTransloco()], + providers: [ + provideRouter([]), + DataMaskingStore, + { provide: DfDataMaskingApiService, useValue: {} }, + { + provide: DfSnackbarService, + useValue: { openSnackBar: jest.fn(), openError: jest.fn() }, + }, + { provide: DmUndoToastService, useValue: { open: jest.fn() } }, + ], + }); + const store = TestBed.inject(DataMaskingStore); + store.loading = false; + const list = Array.from({ length: 300 }, (_, i) => + column({ id: `big.w.c${i}`, service: 'big', table: 'w', column: `c${i}` }) + ); + store.overview = { + mode: 'preview', + scan: { + status: 'idle', + columns: 300, + finishedAt: '2026-10-28T00:00:00Z', + }, + counts: { masked: 300, review: 0, clear: 0 }, + you: { canGrant: true, seesRealValues: false }, + }; + store.columns.set('big', list); + store.services = [ + { service: 'big', label: 'Big', tables: [summarize('w', list)] }, + ]; + const fixture = TestBed.createComponent(DfDmColumnsComponent); + fixture.detectChanges(); + const count = () => + (fixture.nativeElement as HTMLElement).querySelectorAll( + '[data-testid="dm-row"]' + ).length; + expect(count()).toBe(FIRST_ROWS); + tick(16 * 4); + fixture.detectChanges(); + expect(count()).toBe(300); + // Class menus are built when opened, not once per row. + expect(document.querySelectorAll('.dm-class-menu').length).toBe(0); + expect( + (fixture.nativeElement as HTMLElement).querySelectorAll('[mat-menu-item]') + .length + ).toBe(0); + flush(); + })); +}); diff --git a/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.ts b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.ts new file mode 100644 index 000000000..3ca32d844 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.component.ts @@ -0,0 +1,714 @@ +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DoCheck, + ElementRef, + OnDestroy, + OnInit, + QueryList, + ViewChildren, + inject, +} from '@angular/core'; +import { NgClass, NgFor, NgIf, NgTemplateOutlet } from '@angular/common'; +import { FormsModule } from '@angular/forms'; +import { Router } from '@angular/router'; +import { MatButtonModule } from '@angular/material/button'; +import { MatIconModule } from '@angular/material/icon'; +import { MatSelectModule } from '@angular/material/select'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { TranslocoPipe } from '@ngneat/transloco'; +import { Subscription } from 'rxjs'; +import { DfEmptyStateComponent } from 'src/app/shared/components/df-empty-state/df-empty-state.component'; +import { ROUTES } from 'src/app/shared/types/routes'; +import { BASELINES, BaselineDef } from '../data-masking.catalog'; +import { DfClassPickerComponent } from '../df-class-picker/df-class-picker.component'; +import { DfEvidenceLedgerComponent } from '../df-evidence-ledger/df-evidence-ledger.component'; +import { DfMaskedValueComponent } from '../df-masked-value/df-masked-value.component'; +import { DfSensitivityStripComponent } from '../df-sensitivity-strip/df-sensitivity-strip.component'; +import { + ColumnFilter, + DataMaskingStore, + tableKey, +} from '../data-masking.store'; +import { + CoverageService, + MaskedColumn, + ServiceTables, + TableSummary, +} from '../data-masking.types'; +import { relTime } from '../dm-time'; + +interface RailRow { + key: string; + table: string; + summary: TableSummary | undefined; +} + +interface RailGroup { + service: string; + label: string; + rows: RailRow[]; +} + +/** A scanned database with nothing under the current filter. */ +interface QuietRow { + service: string; + label: string; + /** none = tables, nothing sensitive; empty = no tables at all. */ + status: 'none' | 'empty'; +} + +const FILTERS: ColumnFilter[] = ['review', 'masked', 'clear', 'all']; +/** Rows drawn at once; the rest follow in chunks so a wide table opens fast. */ +export const FIRST_ROWS = 60; +const ROW_CHUNK = 120; +const SEARCH_DEBOUNCE_MS = 150; + +/** The filters are disjoint, like the counts on their chips. */ +export function passesFilter(c: MaskedColumn, f: ColumnFilter): boolean { + return f === 'all' || c.verdict === f; +} + +/** + * Columns tab. Change detection is OnPush: ngDoCheck compares the store + * state the view reads (references and counters) and marks the view only + * when one moved, so wide schemas do not re-check every row on every event. + * The rail and the grid are one tab stop each (roving tabindex): arrows + * move inside, Tab moves on. + */ +@Component({ + selector: 'df-dm-columns', + standalone: true, + changeDetection: ChangeDetectionStrategy.OnPush, + imports: [ + NgIf, + NgFor, + NgClass, + NgTemplateOutlet, + FormsModule, + MatButtonModule, + MatIconModule, + MatSelectModule, + MatTooltipModule, + TranslocoPipe, + DfEmptyStateComponent, + DfClassPickerComponent, + DfEvidenceLedgerComponent, + DfMaskedValueComponent, + DfSensitivityStripComponent, + ], + templateUrl: './df-dm-columns.component.html', + styleUrls: ['./df-dm-columns.component.scss'], +}) +export class DfDmColumnsComponent implements OnInit, DoCheck, OnDestroy { + readonly store = inject(DataMaskingStore); + private router = inject(Router); + private cdr = inject(ChangeDetectorRef); + readonly allFilters = FILTERS; + /** First run: one shortcut per framework into its Compliance baseline. */ + readonly baselines = BASELINES; + + @ViewChildren('rowEl') rowEls!: QueryList>; + /** Rail table rows, in railKeys order. */ + @ViewChildren('railEl') railEls!: QueryList>; + + filter: ColumnFilter = 'review'; + search = ''; + selected: string | null = null; + + rail: RailGroup[] = []; + /** Scanned databases with nothing to list under the current filter. */ + quiet: QuietRow[] = []; + /** Column ids shown for the selected table; a decision never removes one. */ + private visible: string[] = []; + rows: MaskedColumn[] = []; + /** How many of `rows` are drawn; grows in chunks after a table opens. */ + shown = FIRST_ROWS; + /** + * Reviewing one column is easier next to its neighbours, so the review + * filter lists the rest of the table after the queue, quietly. + */ + context = new Set(); + /** Index of the first quiet context row, or -1. */ + firstContext = -1; + expanded = new Set(); + /** Rows that just changed state, for the settle animation. */ + settled = new Set(); + /** The grid row that holds the grid's one tab stop. */ + activeRow = 0; + /** The rail row that holds the rail's one tab stop. */ + railFocus: string | null = null; + + private lastVersion = -1; + private lastStructure = -1; + private lastServices: ServiceTables[] | null = null; + private seen: unknown[] = []; + private sub?: Subscription; + private timers = new Set>(); + private growTimer?: ReturnType; + private searchTimer?: ReturnType; + + ngOnInit(): void { + this.sub = this.store.viewReset$.subscribe(() => { + this.reset(); + this.cdr.markForCheck(); + }); + if (!this.store.loading) this.reset(); + } + + ngDoCheck(): void { + const s = this.store; + if ( + s.structure !== this.lastStructure || + s.services !== this.lastServices + ) { + this.lastStructure = s.structure; + this.lastServices = s.services; + this.lastVersion = s.version; + if (!s.loading) this.buildRail(); + } else if (s.version !== this.lastVersion) { + this.lastVersion = s.version; + this.refreshRows(); + this.refreshRail(); + } + // What the view reads: store state, and local state methods may change + // outside a template event (tests, timers, the store's view resets). + const now = [ + s.version, + s.structure, + s.services, + s.overview, + s.idFilter, + s.settling, + s.loading, + s.tableCounts, + this.rail, + this.rows, + this.quiet, + this.filter, + this.selected, + this.shown, + this.activeRow, + this.search, + ]; + if (now.some((v, i) => v !== this.seen[i])) { + this.seen = now; + this.cdr.markForCheck(); + } + } + + ngOnDestroy(): void { + this.sub?.unsubscribe(); + this.timers.forEach(id => clearTimeout(id)); + this.timers.clear(); + } + + /** setTimeout that dies with the component and refreshes the view. */ + private later(fn: () => void, ms = 0): ReturnType { + const id = setTimeout(() => { + this.timers.delete(id); + fn(); + this.cdr.markForCheck(); + }, ms); + this.timers.add(id); + return id; + } + + private cancel(id?: ReturnType): void { + if (id === undefined) return; + clearTimeout(id); + this.timers.delete(id); + } + + // ----------------------------------------------------------- view state + + /** Re-snapshot from the store: filter, rail and selection. */ + reset(): void { + const requested = this.store.filter; + this.store.filter = null; + if (this.store.idFilter) { + this.filter = 'all'; + } else if (requested) { + this.filter = requested; + } else { + this.filter = this.store.counts.review > 0 ? 'review' : 'masked'; + } + this.buildRail(); + } + + setFilter(f: ColumnFilter): void { + this.filter = f; + this.buildRail(); + } + + /** Typing: rebuild once the keystrokes pause. */ + searchChanged(): void { + this.cancel(this.searchTimer); + this.searchTimer = this.later(() => this.onSearch(), SEARCH_DEBOUNCE_MS); + } + + onSearch(): void { + this.cancel(this.searchTimer); + this.buildRail(); + } + + clearSearch(): void { + this.search = ''; + this.onSearch(); + } + + clearIdFilter(): void { + this.store.idFilter = null; + this.buildRail(); + } + + select(key: string): void { + const same = key === this.selected; + this.selected = key; + this.railFocus = key; + this.expanded.clear(); + this.snapshotRows(!same); + } + + private get q(): string { + return this.search.trim().toLowerCase(); + } + + summary(service: string, table: string): TableSummary | undefined { + return this.store.summaryOf(service, table); + } + + summaryFor(key: string): TableSummary | undefined { + const [service, table] = key.split('\u0000'); + return this.summary(service, table); + } + + private columnPasses(c: MaskedColumn, tableHit: boolean): boolean { + const ids = this.store.idFilter?.ids; + if (ids && !ids.has(c.id)) return false; + if (!passesFilter(c, this.filter)) return false; + if (this.q && !tableHit && !c.column.toLowerCase().includes(this.q)) { + return false; + } + return true; + } + + private buildRail(): void { + const q = this.q; + const groups: RailGroup[] = []; + for (const s of this.store.services) { + const rows: RailRow[] = []; + for (const t of s.tables) { + const hit = !!q && t.name.toLowerCase().includes(q); + const cols = this.store.tableColumns(s.service, t.name); + if (!cols.some(c => this.columnPasses(c, hit))) continue; + rows.push(this.railRow(s.service, t.name)); + } + if (rows.length) + groups.push({ service: s.service, label: s.label, rows }); + } + this.rail = groups; + this.quiet = this.quietRows(); + const keys = this.railKeys; + const before = this.selected; + if (!this.selected || !keys.includes(this.selected)) { + this.selected = keys[0] ?? null; + this.expanded.clear(); + } + if (!this.railFocus || !keys.includes(this.railFocus)) { + this.railFocus = this.selected; + } + this.snapshotRows(this.selected !== before); + } + + private railRow(service: string, table: string): RailRow { + return { + key: tableKey(service, table), + table, + summary: this.summary(service, table), + }; + } + + /** A decision changed some summaries: refresh them, keep membership. */ + private refreshRail(): void { + this.rail = this.rail.map(g => ({ + ...g, + rows: g.rows.map(r => { + const [service] = r.key.split('\u0000'); + const summary = this.summary(service, r.table); + return summary === r.summary ? r : { ...r, summary }; + }), + })); + } + + /** + * Scanned databases missing from the rail: one with tables but nothing + * masked or to review (under those filters), or one with no tables. They + * stay listed so a scanned database never looks unscanned. + */ + private quietRows(): QuietRow[] { + if (this.store.idFilter) return []; + const inRail = new Set(this.rail.map(g => g.service)); + const out: QuietRow[] = []; + for (const s of this.store.services) { + if (inRail.has(s.service) || this.store.scanErrorOf(s)) continue; + if (!this.matchesSearch(s.label, s.service)) continue; + if (!s.tables.length) { + if (s.scannedAt) out.push({ ...this.quietOf(s), status: 'empty' }); + continue; + } + const sensitive = s.tables.some( + t => t.columns.masked + t.columns.review > 0 + ); + if ( + !sensitive && + (this.filter === 'masked' || this.filter === 'review') + ) { + out.push({ ...this.quietOf(s), status: 'none' }); + } + } + return out; + } + + private quietOf(s: ServiceTables) { + return { service: s.service, label: s.label }; + } + + /** Open a "nothing sensitive" database under All, on its first table. */ + openQuiet(row: QuietRow): void { + const svc = this.store.services.find(s => s.service === row.service); + const first = svc?.tables[0]?.name; + this.search = ''; + this.filter = 'all'; + this.buildRail(); + if (first) this.select(tableKey(row.service, first)); + } + + /** + * Rebuild the row list for the selected table. `opened` is a different + * table: drawing starts over from the first rows. The same table keeps + * what is drawn (a filter or search change must not redraw it all). + */ + private snapshotRows(opened = true): void { + if (!this.selected) { + this.visible = []; + this.rows = []; + this.context = new Set(); + this.firstContext = -1; + return; + } + const [service, table] = this.selected.split('\u0000'); + const hit = !!this.q && table.toLowerCase().includes(this.q); + const all = this.store.tableColumns(service, table); + const queue = all.filter(c => this.columnPasses(c, hit)).map(c => c.id); + const withContext = + this.filter === 'review' && !this.q && !this.store.idFilter; + const inQueue = new Set(queue); + const rest = withContext + ? all.map(c => c.id).filter(id => !inQueue.has(id)) + : []; + this.context = new Set(rest); + this.visible = [...queue, ...rest]; + if (opened) this.activeRow = 0; + this.refreshRows(); + this.startGrowing(opened); + } + + private refreshRows(): void { + if (!this.selected) { + this.rows = []; + return; + } + const [service, table] = this.selected.split('\u0000'); + const byId = new Map( + this.store.tableColumns(service, table).map(c => [c.id, c]) + ); + this.rows = this.visible + .map(id => byId.get(id)) + .filter((c): c is MaskedColumn => !!c); + this.firstContext = this.rows.findIndex(r => this.context.has(r.id)); + if (this.activeRow >= this.rows.length) this.activeRow = 0; + } + + /** Draw the first rows now and the rest a chunk per tick. */ + private startGrowing(opened: boolean): void { + this.cancel(this.growTimer); + const keep = opened ? 0 : this.shown; + this.shown = Math.min(this.rows.length, Math.max(FIRST_ROWS, keep)); + const grow = () => { + this.shown = Math.min(this.rows.length, this.shown + ROW_CHUNK); + if (this.shown < this.rows.length) this.growTimer = this.later(grow, 16); + }; + if (this.shown < this.rows.length) this.growTimer = this.later(grow, 16); + } + + get shownRows(): MaskedColumn[] { + return this.shown >= this.rows.length + ? this.rows + : this.rows.slice(0, this.shown); + } + + // -------------------------------------------------------------- derived + + get scanProgress(): number { + return this.store.scanProgress ?? 0; + } + + /** Scan button / first-run label: with a percentage once the server has one. */ + get scanningLabel(): string { + const p = this.store.scanProgress; + return p === null + ? this.store.t('columns.scanningNoProgress') + : this.store.t('columns.scanning', { progress: p }); + } + + scan(service?: string): void { + if (!this.store.scanning) this.store.scan(service); + } + + /** + * The server lists no SQL database it could scan (a brand-new instance): + * the first step is adding one, not scanning. + */ + get noDatabases(): boolean { + const c = this.store.overview?.coverage; + return !!c && c.covered.length === 0 && !this.store.services.length; + } + + /** The last (first) scan could not read these databases. */ + get firstScanFailures(): string { + const scan = this.store.overview?.scan; + if (scan?.lastStatus !== 'failed' || !scan.failed?.length) return ''; + return this.store.nameList(scan.failed.map(f => f.label || f.service)); + } + + addDatabase(): void { + this.router.navigate([ + '/', + ROUTES.API_CONNECTIONS, + ROUTES.API_TYPES, + ROUTES.DATABASE, + ROUTES.CREATE, + ]); + } + + /** Open Compliance on a baseline. */ + openBaseline(b: BaselineDef): void { + this.router.navigate(['/', ROUTES.API_SECURITY, ROUTES.COMPLIANCE], { + queryParams: { baseline: b.id }, + }); + } + + private matchesSearch(...names: string[]): boolean { + const q = this.q; + return !q || names.some(n => n.toLowerCase().includes(q)); + } + + /** Covered services that were never scanned: shown with a Scan action. */ + get pendingServices(): ServiceTables[] { + if (this.store.idFilter) return []; + return this.store.unscannedServices.filter(s => + this.matchesSearch(s.label, s.service) + ); + } + + /** Covered services the last scan could not read: shown with Retry. */ + get failedServices(): ServiceTables[] { + if (this.store.idFilter) return []; + return this.store.failedServices.filter(s => + this.matchesSearch(s.label, s.service) + ); + } + + /** Services Data Masking cannot mask: a muted group at the rail's foot. */ + get uncovered(): CoverageService[] { + if (this.store.idFilter) return []; + return this.store.uncovered.filter(s => + this.matchesSearch(s.label, s.service, s.type) + ); + } + + get railKeys(): string[] { + return this.rail.flatMap(g => g.rows.map(r => r.key)); + } + + get selectedTable(): string { + return this.selected?.split('\u0000')[1] ?? ''; + } + + get selectedService(): string { + const svc = this.selected?.split('\u0000')[0]; + return this.store.services.find(s => s.service === svc)?.label ?? ''; + } + + /** Chips: To review only while there is something to review (or it is open). */ + get filters(): ColumnFilter[] { + return FILTERS.filter( + f => f !== 'review' || this.count('review') > 0 || this.filter === f + ); + } + + /** Disjoint counts that add up to All, like the header and the export. */ + count(f: ColumnFilter): number { + const c = this.store.counts; + return f === 'all' ? c.masked + c.review + c.clear : c[f]; + } + + /** Review flow: the next rail table that still has columns to review. */ + get nextReviewTable(): string | null { + if (this.filter !== 'review') return null; + const keys = this.railKeys; + const i = keys.indexOf(this.selected ?? ''); + const ordered = [...keys.slice(i + 1), ...keys.slice(0, Math.max(i, 0))]; + return ( + ordered.find(k => (this.summaryFor(k)?.columns.review ?? 0) > 0) ?? null + ); + } + + get selectedDone(): boolean { + return ( + this.filter === 'review' && + !!this.selected && + (this.summaryFor(this.selected)?.columns.review ?? 0) === 0 + ); + } + + stripLabel(t: TableSummary | undefined): string { + if (!t) return ''; + return this.store.t('strip', { + masked: t.columns.masked, + review: t.columns.review, + clear: t.columns.clear, + }); + } + + decidedTip(c: MaskedColumn): string { + const when = relTime(c.decidedAt); + return c.decidedBy + ? this.store.t('ledger.setBy', { name: c.decidedBy, when }) + : this.store.t('ledger.setByYou', { when }); + } + + // -------------------------------------------------------------- actions + + /** + * Decide a column. From a row button, focus moves to the row first: the + * button disappears with the decision, and keyboard review (j/k/m/n) + * continues from the row. + */ + decide(c: MaskedColumn, cls: string | null, row?: HTMLElement): void { + if (row) { + const i = this.rows.findIndex(r => r.id === c.id); + if (i >= 0) this.activeRow = i; + row.focus(); + } + this.store.decide(c, cls); + this.settled.add(c.id); + this.later(() => this.settled.delete(c.id), 900); + } + + /** + * Keep masked confirms the detected class. A review column without one + * has nothing to keep (and null would mean not sensitive), so the button + * is hidden and the class chip is the way to decide. + */ + canKeep(c: MaskedColumn): boolean { + return c.verdict === 'review' && !!c.class; + } + + keep(c: MaskedColumn, row?: HTMLElement): void { + if (this.canKeep(c)) this.decide(c, c.class, row); + } + + toggle(c: MaskedColumn): void { + if (this.expanded.has(c.id)) this.expanded.delete(c.id); + else this.expanded.add(c.id); + } + + goNext(): void { + const next = this.nextReviewTable; + if (next) { + this.select(next); + this.later(() => this.focusRow(0)); + } + } + + private focusRow(i: number): void { + const target = Math.max(0, Math.min(i, this.rows.length - 1)); + this.activeRow = target; + if (target >= this.shown) { + this.shown = Math.min(this.rows.length, target + ROW_CHUNK); + this.cdr.detectChanges(); + } + this.rowEls?.toArray()[target]?.nativeElement.focus(); + } + + /** j/k (arrows, Home/End) move, m keeps masked, n marks not sensitive, enter expands. */ + onRowKey(e: KeyboardEvent, c: MaskedColumn, i: number): void { + if (e.target !== e.currentTarget) return; + switch (e.key) { + case 'j': + case 'ArrowDown': + this.focusRow(i + 1); + break; + case 'k': + case 'ArrowUp': + this.focusRow(i - 1); + break; + case 'Home': + this.focusRow(0); + break; + case 'End': + this.focusRow(this.rows.length - 1); + break; + case 'm': + this.keep(c); + break; + case 'n': + if (c.verdict === 'review') this.decide(c, null); + break; + case 'Enter': + this.toggle(c); + break; + default: + return; + } + e.preventDefault(); + } + + /** Arrows (and Home/End) move through the rail; the focused table opens. */ + onRailKey(e: KeyboardEvent, key: string): void { + const keys = this.railKeys; + const i = keys.indexOf(key); + let next: string | undefined; + switch (e.key) { + case 'ArrowDown': + next = keys[Math.min(i + 1, keys.length - 1)]; + break; + case 'ArrowUp': + next = keys[Math.max(i - 1, 0)]; + break; + case 'Home': + next = keys[0]; + break; + case 'End': + next = keys[keys.length - 1]; + break; + default: + return; + } + e.preventDefault(); + if (!next || next === key) return; + this.select(next); + this.cdr.detectChanges(); + this.railEls?.toArray()[keys.indexOf(next)]?.nativeElement.focus(); + } + + trackRailRow = (_: number, r: RailRow) => r.key; + trackGroup = (_: number, g: RailGroup) => g.service; + trackService = (_: number, s: { service: string }) => s.service; + trackBaseline = (_: number, b: BaselineDef) => b.id; + trackRow = (_: number, c: MaskedColumn) => c.id; + keyOf = (service: string, table: string) => tableKey(service, table); +} diff --git a/src/app/adf-data-masking/df-dm-columns/df-dm-columns.first-run.spec.ts b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.first-run.spec.ts new file mode 100644 index 000000000..8395aa73d --- /dev/null +++ b/src/app/adf-data-masking/df-dm-columns/df-dm-columns.first-run.spec.ts @@ -0,0 +1,452 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { Router, provideRouter } from '@angular/router'; +import { of } from 'rxjs'; +import { DfDmColumnsComponent } from './df-dm-columns.component'; +import { DataMaskingStore, summarize } from '../data-masking.store'; +import { DfDataMaskingApiService } from '../data-masking-api.service'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { column, dmTransloco } from '../dm-testing'; +import { MaskingOverview } from '../data-masking.types'; +import { DmUndoToastService } from '../df-dm-undo-toast/df-dm-undo-toast.component'; + +const overview = (over: Partial = {}): MaskingOverview => ({ + mode: 'off', + scan: { status: 'idle', progress: null, finishedAt: null, columns: 0 }, + counts: { masked: 0, review: 0, clear: 0 }, + you: { canGrant: true, seesRealValues: false }, + coverage: { + covered: [{ service: 'crm', label: 'CRM', type: 'pgsql' }], + uncovered: [ + { service: 'events', label: 'Event store', type: 'mongodb' }, + { service: 'cache', label: 'Session cache', type: 'couchdb' }, + ], + }, + ...over, +}); + +describe('DfDmColumnsComponent (first run and coverage)', () => { + let fixture: ComponentFixture; + let store: DataMaskingStore; + let api: { scan: jest.Mock; overview: jest.Mock }; + + const el = () => fixture.nativeElement as HTMLElement; + const q = (id: string) => + el().querySelector(`[data-testid="${id}"]`) as HTMLElement | null; + + beforeEach(() => { + api = { + scan: jest.fn(() => + of(overview({ scan: { status: 'running', progress: 0, columns: 0 } })) + ), + overview: jest.fn(() => of(overview())), + }; + TestBed.configureTestingModule({ + imports: [DfDmColumnsComponent, NoopAnimationsModule, dmTransloco()], + providers: [ + provideRouter([]), + DataMaskingStore, + { provide: DfDataMaskingApiService, useValue: api }, + { + provide: DfSnackbarService, + useValue: { openSnackBar: jest.fn(), openError: jest.fn() }, + }, + { provide: DmUndoToastService, useValue: { open: jest.fn() } }, + ], + }); + store = TestBed.inject(DataMaskingStore); + store.loading = false; + store.overview = overview(); + // The real /table lists covered services before any scan, empty. + store.services = [{ service: 'crm', label: 'CRM', tables: [] }]; + fixture = TestBed.createComponent(DfDmColumnsComponent); + fixture.detectChanges(); + }); + + it('a covered service with no tables is still the never-scanned state', () => { + expect(store.scanned).toBe(false); + expect(store.unscanned).toBe(true); + expect(q('dm-no-scan')).toBeTruthy(); + expect(q('dm-search')).toBeNull(); + }); + + it('first run offers one primary action and a baseline per framework', () => { + const buttons = el().querySelectorAll('[data-testid="empty-state-action"]'); + expect(buttons.length).toBe(1); + expect(buttons[0].textContent).toContain('Scan databases'); + const chips = Array.from( + el().querySelectorAll('[data-testid^="dm-baseline-"]') + ).map(b => b.textContent?.replace(/\s+/g, ' ').trim()); + expect(chips).toEqual([ + 'health_and_safety HIPAA', + 'credit_card PCI DSS', + 'account_balance GLBA', + 'gavel CJIS', + 'privacy_tip GDPR', + 'verified_user SOC 2', + 'workspace_premium ISO 27001', + ]); + }); + + it('a baseline chip opens Compliance on that baseline', () => { + const router = TestBed.inject(Router); + const nav = jest.spyOn(router, 'navigate').mockResolvedValue(true); + q('dm-baseline-pci_dss')!.click(); + expect(nav).toHaveBeenCalledWith(['/', 'api-security', 'compliance'], { + queryParams: { baseline: 'pci_dss' }, + }); + }); + + it('with no SQL database to scan, the first step is adding one', () => { + const router = TestBed.inject(Router); + const nav = jest.spyOn(router, 'navigate').mockResolvedValue(true); + store.overview = overview({ coverage: { covered: [], uncovered: [] } }); + store.services = []; + fixture.detectChanges(); + expect(q('dm-no-scan')).toBeNull(); + expect(q('dm-no-databases')!.textContent).toContain('No SQL databases yet'); + // Baselines still apply ahead of any data. + expect(q('dm-first-run-baselines')).toBeTruthy(); + q('empty-state-action')!.click(); + expect(nav).toHaveBeenCalledWith([ + '/', + 'api-connections', + 'api-types', + 'database', + 'create', + ]); + expect(api.scan).not.toHaveBeenCalled(); + }); + + it('a default install whose only database has no tables is sent to add one, not to scan', () => { + store.overview = overview({ + coverage: { + covered: [ + { service: 'db', label: 'Local SQL Database', type: 'sqlite' }, + ], + uncovered: [], + }, + }); + store.services = [ + { service: 'db', label: 'Local SQL Database', tables: [] }, + ]; + store.tableCounts = new Map([['db', 0]]); + fixture.detectChanges(); + expect(q('dm-no-scan')).toBeNull(); + expect(q('dm-no-databases')!.textContent).toContain('No tables to scan'); + expect(q('empty-state-action')!.textContent).toContain('Add a database'); + }); + + it('says which database a first scan could not read', () => { + store.overview = overview({ + scan: { + status: 'idle', + progress: null, + finishedAt: null, + columns: 0, + lastStatus: 'failed', + failed: [{ service: 'crm', label: 'CRM', reason: 'refused' }], + }, + }); + fixture.detectChanges(); + expect(q('dm-first-scan-failed')!.textContent).toContain( + "Scan failed. Couldn't read CRM." + ); + expect(q('dm-no-scan')).toBeTruthy(); + }); + + it('the first scan shows progress in place of the call to action', () => { + q('empty-state-action')!.click(); + fixture.detectChanges(); + expect(api.scan).toHaveBeenCalledWith(undefined); + expect(q('dm-no-scan')).toBeNull(); + const panel = q('dm-first-scan')!; + expect(panel).toBeTruthy(); + expect(panel.textContent).toContain('Scanning 0%'); + // No empty filter flashes while the first scan runs. + expect(el().textContent).not.toContain('No masked columns'); + expect(el().textContent).not.toContain('All reviewed'); + }); + + it('progress without a percentage reads as scanning, not 0%', () => { + store.overview = overview({ + scan: { status: 'running', progress: null, columns: 0 }, + }); + fixture.detectChanges(); + expect(q('dm-first-scan')!.textContent).toContain('Scanning…'); + expect( + el() + .querySelector('.first-scan-bar')! + .classList.contains('is-indeterminate') + ).toBe(true); + }); + + describe('after a scan', () => { + beforeEach(() => { + const list = [column({ id: 'crm.a.ssn', table: 'a', column: 'ssn' })]; + store.columns.set('crm', list); + store.services = [ + { service: 'crm', label: 'CRM', tables: [summarize('a', list)] }, + // Added after the last scan: covered, never scanned. + { service: 'hr', label: 'HR (SQL Server)', tables: [] }, + ]; + store.overview = overview({ + mode: 'preview', + scan: { + status: 'idle', + progress: 100, + finishedAt: '2026-10-28T00:00:00Z', + columns: 1, + }, + counts: { masked: 1, review: 0, clear: 0 }, + }); + fixture.componentInstance.reset(); + fixture.detectChanges(); + }); + + it('lists uncovered services last, muted and not selectable', () => { + const group = q('dm-rail-uncovered')!; + expect(group).toBeTruthy(); + expect(group.querySelector('h3')?.textContent).toContain('Not covered'); + const rows = Array.from( + group.querySelectorAll('[data-testid="dm-rail-uncovered-row"]') + ); + expect(rows.map(r => r.querySelector('.rail-name')?.textContent)).toEqual( + ['Event store', 'Session cache'] + ); + // Not a button and never part of the table selection. + expect(rows[0].tagName).toBe('LI'); + expect(group.querySelector('button')).toBeNull(); + // The explanation is spoken, not only hovered. + expect(rows[0].textContent).toContain("Data Masking can't mask"); + // It is the last group in the rail. + const rail = el().querySelector('nav.rail')!; + expect(rail.lastElementChild).toBe(group); + }); + + it('offers a per-service scan for a covered service never scanned', () => { + const pending = q('dm-rail-unscanned')!; + expect(pending.textContent).toContain('HR (SQL Server)'); + expect(pending.textContent).toContain('Not scanned yet'); + (pending.querySelector('button') as HTMLButtonElement).click(); + expect(api.scan).toHaveBeenCalledWith('hr'); + }); + + it('search narrows the uncovered list too', () => { + const cmp = fixture.componentInstance; + cmp.search = 'couch'; + cmp.onSearch(); + expect(cmp.uncovered.map(u => u.service)).toEqual(['cache']); + }); + + it('a rule-matches filter hides coverage rows', () => { + store.idFilter = { label: 'x', ids: new Set(['crm.a.ssn']) }; + expect(fixture.componentInstance.uncovered).toEqual([]); + expect(fixture.componentInstance.pendingServices).toEqual([]); + }); + }); + + describe('scanned databases with nothing to list', () => { + beforeEach(() => { + const list = [column({ id: 'crm.a.ssn', table: 'a', column: 'ssn' })]; + const inv = [ + column({ + id: 'inv.w.sku', + service: 'inv', + table: 'w', + column: 'sku', + verdict: 'clear', + class: null, + tier: null, + }), + ]; + store.columns.set('crm', list); + store.columns.set('inv', inv); + store.services = [ + { + service: 'crm', + label: 'CRM', + scannedAt: 'x', + tables: [summarize('a', list)], + }, + { + service: 'inv', + label: 'Inventory', + scannedAt: 'x', + tables: [summarize('w', inv)], + }, + { + service: 'db', + label: 'Local SQL Database', + scannedAt: 'x', + tables: [], + }, + { + service: 'hr', + label: 'HR', + scannedAt: null, + scanError: 'Login failed for user', + tables: [], + }, + ]; + store.overview = overview({ + mode: 'preview', + scan: { status: 'idle', finishedAt: 'x', columns: 2 }, + counts: { masked: 1, review: 0, clear: 1 }, + }); + fixture.componentInstance.reset(); + fixture.detectChanges(); + }); + + it('stay listed under Masked: nothing sensitive, no tables', () => { + const rows = Array.from( + el().querySelectorAll('[data-testid="dm-rail-quiet-row"]') + ).map( + r => + `${r.querySelector('.rail-name')?.textContent?.trim()}: ${r + .querySelector('.rail-note') + ?.textContent?.trim()}` + ); + expect(rows).toEqual([ + 'Inventory: Nothing sensitive', + 'Local SQL Database: No tables', + ]); + }); + + it('a database the scan could not read is listed with Retry', () => { + const failed = q('dm-rail-failed')!; + expect(failed.textContent).toContain("Couldn't scan"); + expect(failed.textContent).toContain('HR'); + expect(failed.textContent).toContain('Login failed for user'); + expect(q('dm-rail-unscanned')).toBeNull(); + (failed.querySelector('button') as HTMLButtonElement).click(); + expect(api.scan).toHaveBeenCalledWith('hr'); + }); + + it('opening a nothing-sensitive database shows it under All', () => { + ( + el().querySelector( + 'button[data-testid="dm-rail-quiet-row"]' + ) as HTMLElement + ).click(); + fixture.detectChanges(); + const cmp = fixture.componentInstance; + expect(cmp.filter).toBe('all'); + expect(cmp.selectedTable).toBe('w'); + }); + + it('new results keep the filter and the selected table', () => { + const cmp = fixture.componentInstance; + cmp.setFilter('all'); + cmp.select(cmp.railKeys[1]); + const list = store.columns.get('crm')!; + store.services = [ + ...store.services, + { service: 'new', label: 'New', scannedAt: null, tables: [] }, + ]; + store.structure++; + void list; + fixture.detectChanges(); + expect(cmp.filter).toBe('all'); + expect(cmp.selectedTable).toBe('w'); + expect(q('dm-rail-unscanned')!.textContent).toContain('New'); + }); + }); + + it('after a scan that found no tables anywhere, the next step is adding a database', () => { + store.services = [ + { + service: 'db', + label: 'Local SQL Database', + scannedAt: 'x', + tables: [], + }, + ]; + store.overview = overview({ + mode: 'preview', + scan: { status: 'idle', finishedAt: 'x', columns: 0 }, + }); + fixture.detectChanges(); + expect(q('dm-no-tables')!.textContent).toContain('No tables to scan'); + expect(q('dm-search')).toBeNull(); + }); + + describe('when the server could not read the scan history', () => { + const LINE = "Couldn't read the scan history. Check the DreamFactory log."; + + it('says so in place of the never-scanned state, with no scan action', () => { + store.overview = overview({ + scan: { + status: 'idle', + finishedAt: null, + columns: 0, + lastStatus: null, + failed: [], + unknown: true, + }, + }); + store.services = [ + { service: 'crm', label: 'CRM', scannedAt: null, tables: [] }, + ]; + fixture.detectChanges(); + expect(store.scanUnknown).toBe(true); + expect(q('dm-scan-unknown')!.textContent).toContain(LINE); + expect(q('dm-no-scan')).toBeNull(); + expect(el().textContent).not.toContain('No scan yet'); + expect(el().textContent).not.toContain('Scan databases'); + expect( + el().querySelector('[data-testid="empty-state-action"]') + ).toBeNull(); + // The rest of the first-run screen still works. + expect(q('dm-first-run-baselines')).toBeTruthy(); + }); + + it("an entry's scan_unknown alone is enough", () => { + store.services = [ + { + service: 'crm', + label: 'CRM', + scannedAt: null, + scanUnknown: true, + tables: [], + }, + ]; + fixture.detectChanges(); + expect(q('dm-scan-unknown')).toBeTruthy(); + expect(q('dm-no-scan')).toBeNull(); + }); + + it('a database with no classified tables is not "Not scanned yet"', () => { + const list = [column({ id: 'crm.a.ssn', table: 'a', column: 'ssn' })]; + store.columns.set('crm', list); + store.services = [ + { + service: 'crm', + label: 'CRM', + scannedAt: null, + scanUnknown: true, + tables: [summarize('a', list)], + }, + { + service: 'hr', + label: 'HR (SQL Server)', + scannedAt: null, + scanUnknown: true, + tables: [], + }, + ]; + store.overview = overview({ + mode: 'preview', + scan: { status: 'idle', finishedAt: null, columns: 1, unknown: true }, + counts: { masked: 1, review: 0, clear: 0 }, + }); + fixture.componentInstance.reset(); + fixture.detectChanges(); + // Results still show; the rail says the history is unreadable. + expect(q('dm-search')).toBeTruthy(); + expect(q('dm-rail-unscanned')).toBeNull(); + expect(q('dm-scan-unknown')!.textContent).toContain(LINE); + expect(el().textContent).not.toContain('Not scanned yet'); + }); + }); +}); diff --git a/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.html b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.html new file mode 100644 index 000000000..8090308b3 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.html @@ -0,0 +1,144 @@ + + + + + + +
    +
    +

    {{ 'dataMasking.detection.title' | transloco }}

    + +
    + +
    +
    + {{ + 'dataMasking.detection.match' | transloco + }} + {{ + 'dataMasking.detection.pattern' | transloco + }} + {{ + 'dataMasking.columns.class' | transloco + }} + {{ + 'dataMasking.detection.matches' | transloco + }} + +
    +
    + {{ + (r.match === 'name' + ? 'dataMasking.detection.matchName' + : 'dataMasking.detection.matchValue' + ) | transloco + }} + {{ r.pattern }}{{ + 'dataMasking.detection.fromBaseline' | transloco: { name: b } + }} + + + {{ 'dataMasking.classes.' + r.class | transloco }} + + + + + + + +
    +
    + + + + + +
    + +
    +
    +

    + + {{ 'dataMasking.tiers.' + g.tier | transloco }} +

    +
    + {{ + 'dataMasking.classes.' + c.id | transloco + }} + + {{ + d + }} + + +
    +
    +
    +
    +
    +     diff --git a/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.scss b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.scss new file mode 100644 index 000000000..32dc44e30 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.scss @@ -0,0 +1,240 @@ +@use '../dm-tier' as *; +@include dm-tiers; + +:host { + display: block; +} + +.detection { + max-width: 920px; +} + +.head { + align-items: center; + display: flex; + justify-content: space-between; + margin-bottom: var(--df-space-3); +} + +.title { + color: var(--df-text); + font-size: var(--df-font-size-lg); + font-weight: var(--df-font-weight-heading); + margin: 0; +} + +.row { + align-items: center; + border-bottom: 1px solid var(--df-border-2); + display: grid; + gap: var(--df-space-3); + grid-template-columns: 120px minmax(0, 1.4fr) minmax(0, 1fr) 80px 40px; + min-height: var(--df-row-height); + padding: 0 var(--df-space-2); + + &.head-row { + border-top: 1px solid var(--df-border-2); + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); + font-weight: var(--df-font-weight-medium); + min-height: 32px; + } +} + +.match { + color: var(--df-text-2); + font-size: var(--df-font-size-sm); +} + +.pattern-cell { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; + padding: var(--df-space-1) 0; +} + +.pattern { + color: var(--df-text); + font-family: var(--df-font-mono); + font-size: var(--df-font-size-xs); + overflow-wrap: anywhere; +} + +// Which Compliance baseline added the rule (Apply adds vertical rules). +.source { + color: var(--df-text-muted); + font-size: var(--df-font-size-xs); +} + +.cls { + align-items: center; + color: var(--df-text); + display: flex; + font-size: var(--df-font-size-sm); + gap: var(--df-space-2); +} + +.dot { + background: var(--dm-tier-fill); + border-radius: 50%; + flex-shrink: 0; + height: 8px; + width: 8px; +} + +.num { + justify-self: end; + text-align: right; +} + +.bi-none { + color: var(--df-text-faint); +} + +.count { + background: none; + border: 0; + border-radius: var(--df-radius-sm); + color: var(--df-accent); + cursor: pointer; + font: inherit; + font-size: var(--df-font-size-sm); + font-weight: var(--df-font-weight-medium); + padding: 2px 6px; + text-decoration: underline; + text-decoration-color: var(--df-accent-soft); + text-underline-offset: 3px; + + &:hover:not(:disabled) { + background: var(--df-accent-soft); + } + + &:disabled { + color: var(--df-text-muted); + cursor: default; + text-decoration: none; + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } +} + +.del { + color: var(--df-text-muted); + display: flex; + justify-content: flex-end; +} + +.builtin { + margin-top: var(--df-space-5); +} + +.builtin-toggle { + align-items: center; + background: none; + border: 0; + border-radius: var(--df-radius-sm); + color: var(--df-text-2); + cursor: pointer; + display: inline-flex; + font: inherit; + font-size: var(--df-font-size-sm); + gap: var(--df-space-1); + padding: var(--df-space-1) var(--df-space-2) var(--df-space-1) 0; + + &:hover { + color: var(--df-text); + } + + &:focus-visible { + box-shadow: var(--df-focus-ring); + outline: none; + } + + mat-icon { + color: var(--df-text-muted); + transition: transform var(--df-duration-standard) var(--df-ease-standard); + } + + &[aria-expanded='true'] mat-icon { + transform: rotate(90deg); + } +} + +.ruleset { + color: var(--df-text-muted); + font-family: var(--df-font-mono); + font-size: var(--df-font-size-xs); + margin-left: var(--df-space-1); +} + +.builtin-body { + display: grid; + gap: var(--df-space-4); + padding: var(--df-space-3) 0 0 var(--df-space-6); +} + +.bi-head { + align-items: center; + color: var(--df-text); + display: flex; + font-size: var(--df-font-size-sm); + font-weight: var(--df-font-weight-heading); + gap: var(--df-space-2); + margin: 0 0 var(--df-space-1); +} + +.bi-row { + align-items: baseline; + display: grid; + gap: var(--df-space-3); + grid-template-columns: 200px minmax(0, 1fr); + padding: 3px 0; +} + +.bi-cls { + color: var(--df-text-2); + font-size: var(--df-font-size-sm); +} + +.bi-dets { + display: flex; + flex-wrap: wrap; + gap: var(--df-space-1); +} + +.det { + background: var(--df-surface-2); + border: 1px solid var(--df-border-2); + border-radius: var(--df-radius-sm); + color: var(--df-text-2); + font-size: var(--df-font-size-xs); + padding: 0 6px; +} + +@media (max-width: 760px) { + .row { + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr) auto 40px; + padding: var(--df-space-2); + + > :nth-child(1) { + display: none; + } + } + + .row.head-row { + display: none; + } + + .builtin-body { + padding-left: 0; + } + + .bi-row { + grid-template-columns: minmax(0, 1fr); + gap: var(--df-space-1); + } +} diff --git a/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.spec.ts b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.spec.ts new file mode 100644 index 000000000..bd71cc2c1 --- /dev/null +++ b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.spec.ts @@ -0,0 +1,266 @@ +import { + ComponentFixture, + TestBed, + fakeAsync, + tick, +} from '@angular/core/testing'; +import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { + MAT_DIALOG_DATA, + MatDialog, + MatDialogRef, +} from '@angular/material/dialog'; +import { HttpErrorResponse } from '@angular/common/http'; +import { of, throwError } from 'rxjs'; +import { normalizeError } from 'src/app/shared/utilities/app-error'; +import { DfDmDetectionComponent } from './df-dm-detection.component'; +import { + DfDmRuleDialogComponent, + validPattern, +} from './df-dm-rule-dialog.component'; +import { DataMaskingStore, summarize } from '../data-masking.store'; +import { column } from '../dm-testing'; +import { DfDataMaskingApiService } from '../data-masking-api.service'; +import { DfSnackbarService } from 'src/app/shared/services/df-snackbar.service'; +import { dmTransloco } from '../dm-testing'; +import { DetectionRule } from '../data-masking.types'; + +describe('DfDmDetectionComponent', () => { + let fixture: ComponentFixture; + + const rules: DetectionRule[] = [ + { + id: 1, + match: 'name', + pattern: '^emp_ssn$', + class: 'national_id', + matches: 2, + source: 'admin', + }, + { + id: 2, + match: 'name', + pattern: '^mrn$', + class: 'special_category', + matches: 0, + source: 'baseline:hipaa', + }, + { + id: 3, + match: 'value', + pattern: '^X\\d+$', + class: 'passport_number', + matches: 0, + source: 'baseline:future_one', + }, + ]; + + let dialog: { open: jest.Mock }; + let choice: unknown; + let api: Record; + + beforeEach(() => { + choice = true; + dialog = { open: jest.fn(() => ({ afterClosed: () => of(choice) })) }; + api = { + rules: jest.fn(() => of(rules)), + builtInRules: jest.fn(() => + of({ + ruleset: 'dreamfactory-core', + version: '2026.11.0', + classes: [], + }) + ), + deleteRule: jest.fn(() => of({})), + decide: jest.fn(() => of({})), + overview: jest.fn(() => of(null)), + tables: jest.fn(() => of([])), + columns: jest.fn(() => of([])), + }; + TestBed.configureTestingModule({ + imports: [DfDmDetectionComponent, NoopAnimationsModule, dmTransloco()], + providers: [ + DataMaskingStore, + { provide: DfDataMaskingApiService, useValue: api }, + { + provide: DfSnackbarService, + useValue: { openSnackBar: jest.fn(), openError: jest.fn() }, + }, + ], + }); + TestBed.overrideProvider(MatDialog, { useValue: dialog }); + fixture = TestBed.createComponent(DfDmDetectionComponent); + fixture.detectChanges(); + }); + + it('says which baseline added a rule; admin rules carry no label', () => { + const rows = Array.from( + (fixture.nativeElement as HTMLElement).querySelectorAll( + '.rules .row:not(.head-row)' + ) + ); + expect(rows).toHaveLength(3); + const source = (row: Element) => + row.querySelector('[data-testid="dm-rule-source"]')?.textContent?.trim(); + expect(source(rows[0])).toBeUndefined(); + expect(source(rows[1])).toBe('Added by the HIPAA baseline'); + // An id the UI does not know yet still reads sensibly. + expect(source(rows[2])).toBe('Added by the future_one baseline'); + }); + + describe('deleting a rule', () => { + const ev = (lines: [number, string][]) => ({ + total: 100, + thresholdMask: 70, + thresholdReview: 40, + lines: lines.map(([points, text]) => ({ + points, + kind: 'context' as const, + text, + })), + }); + beforeEach(() => { + const store = TestBed.inject(DataMaskingStore); + const cols = [ + column({ + id: 'crm.a.emp_ssn', + table: 'a', + evidence: ev([[70, 'Custom rule ^emp_ssn$']]), + }), + column({ + id: 'crm.a.ssn', + table: 'a', + evidence: ev([ + [70, 'Custom rule ^emp_ssn$'], + [45, 'Column name'], + ]), + }), + ]; + store.columns.set('crm', cols); + store.services = [ + { service: 'crm', label: 'CRM', tables: [summarize('a', cols)] }, + ]; + }); + + it('asks first, and says its columns stay masked', () => { + choice = false; + fixture.componentInstance.deleteRule(rules[0]); + const data = dialog.open.mock.calls[0][1].data; + expect(data.title).toBe('Delete rule?'); + expect(data.message).toBe('2 columns it masked stay masked.'); + expect(data.extra).toBe('Delete and unmask 1'); + expect(api['deleteRule']).not.toHaveBeenCalled(); + }); + + it('Delete keeps the columns masked', () => { + fixture.componentInstance.deleteRule(rules[0]); + expect(api['deleteRule']).toHaveBeenCalledWith(1); + expect(api['decide']).not.toHaveBeenCalled(); + }); + + it('Delete and unmask clears only the columns nothing else flags', () => { + choice = 'extra'; + fixture.componentInstance.deleteRule(rules[0]); + expect(api['deleteRule']).toHaveBeenCalledWith(1); + expect(api['decide']).toHaveBeenCalledTimes(1); + expect(api['decide']).toHaveBeenCalledWith('crm.a.emp_ssn', null); + }); + }); +}); + +describe('DfDmRuleDialogComponent', () => { + it.each([ + ['atomic group (backtracking-safe)', '^(?>emp|staff)_ssn$'], + ['possessive quantifier', '^\\d++-\\d++$'], + ['inline flag', '(?i)^ssn$'], + ['PCRE named group', '^(?P
    emp)_ssn$'],
+  ])('accepts a valid PCRE pattern: %s', (_label, p) => {
+    expect(validPattern(p)).toBe(true);
+  });
+
+  it("shows the server's reason when the preview pattern does not compile", fakeAsync(() => {
+    // What the error interceptor hands a toast-off caller: a normalized 400.
+    const refused = normalizeError(
+      new HttpErrorResponse({
+        status: 400,
+        error: {
+          error: {
+            code: 400,
+            message: "Pattern doesn't compile: lookbehind is not fixed length",
+          },
+        },
+      })
+    );
+    const testRule = jest.fn(() => throwError(() => refused));
+    TestBed.configureTestingModule({
+      imports: [DfDmRuleDialogComponent, NoopAnimationsModule, dmTransloco()],
+      providers: [
+        { provide: DfDataMaskingApiService, useValue: { testRule } },
+        { provide: MatDialogRef, useValue: { close: jest.fn() } },
+      ],
+    });
+    const f = TestBed.createComponent(DfDmRuleDialogComponent);
+    f.detectChanges();
+    const cmp = f.componentInstance;
+    cmp.draft = { match: 'name', pattern: '(?<=e+)ssn', class: 'national_id' };
+    cmp.changed();
+    tick(300);
+    f.detectChanges();
+    const preview = (f.nativeElement as HTMLElement).querySelector(
+      '[data-testid="dm-rule-preview"]'
+    )!;
+    expect(preview.textContent).toContain('lookbehind is not fixed length');
+    expect(preview.textContent).not.toContain('Matches 0 columns');
+    expect(cmp.canSave).toBe(false);
+    // Editing clears it until the server answers again.
+    cmp.draft = { ...cmp.draft, pattern: 'ssn' };
+    testRule.mockReturnValue(of({ matches: [{ id: 'a.b.ssn' }] }) as never);
+    cmp.changed();
+    tick(300);
+    f.detectChanges();
+    expect(cmp.canSave).toBe(true);
+    expect(preview.textContent).toContain('Matches 1 column');
+  }));
+
+  it.each([
+    [403, 'Only a system administrator can change Data Masking settings.', ''],
+    [400, 'Pattern is too broad', 'Pattern is too broad'],
+  ])(
+    'save refused with %s: the dialog stays open; only a refused rule explains itself in place',
+    (status, message, inline) => {
+      const refused = normalizeError(
+        new HttpErrorResponse({
+          status,
+          error: { error: { code: status, message } },
+        })
+      );
+      const close = jest.fn();
+      TestBed.configureTestingModule({
+        imports: [DfDmRuleDialogComponent, NoopAnimationsModule, dmTransloco()],
+        providers: [
+          {
+            provide: DfDataMaskingApiService,
+            useValue: { testRule: jest.fn(() => of({ matches: [] })) },
+          },
+          { provide: MatDialogRef, useValue: { close } },
+          {
+            provide: MAT_DIALOG_DATA,
+            useValue: { save: () => throwError(() => refused) },
+          },
+        ],
+      });
+      const f = TestBed.createComponent(DfDmRuleDialogComponent);
+      f.detectChanges();
+      const cmp = f.componentInstance;
+      cmp.draft = { match: 'name', pattern: '^ssn$', class: 'national_id' };
+      cmp.save();
+      f.detectChanges();
+      expect(close).not.toHaveBeenCalled();
+      expect(cmp.saving).toBe(false);
+      // A 403 is the store's error toast (the server's message), not the
+      // rule's fault: nothing in place, and Save works again.
+      expect(cmp.saveError ?? '').toBe(inline);
+      expect(cmp.canSave).toBe(true);
+    }
+  );
+});
diff --git a/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.ts b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.ts
new file mode 100644
index 000000000..21c18d231
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-detection/df-dm-detection.component.ts
@@ -0,0 +1,131 @@
+import { Component, EventEmitter, OnInit, Output, inject } from '@angular/core';
+import { NgClass, NgFor, NgIf } from '@angular/common';
+import { MatButtonModule } from '@angular/material/button';
+import { MatDialog, MatDialogModule } from '@angular/material/dialog';
+import { MatIconModule } from '@angular/material/icon';
+import { MatTooltipModule } from '@angular/material/tooltip';
+import { TranslocoPipe } from '@ngneat/transloco';
+import { DfEmptyStateComponent } from 'src/app/shared/components/df-empty-state/df-empty-state.component';
+import { DfSkeletonComponent } from 'src/app/shared/components/df-skeleton/df-skeleton.component';
+import { DfDataMaskingApiService } from '../data-masking-api.service';
+import { DataMaskingStore } from '../data-masking.store';
+import { baselineDef, classDef, classesByTier } from '../data-masking.catalog';
+import { DetectionRule } from '../data-masking.types';
+import {
+  DfDmRuleDialogComponent,
+  RuleDialogData,
+} from './df-dm-rule-dialog.component';
+import {
+  DfDmConfirmDialogComponent,
+  DmConfirmData,
+} from '../df-data-masking/df-dm-confirm-dialog.component';
+
+/**
+ * Detection tab: custom rules on top of the built-in ruleset. A rule's
+ * Matches count opens Columns filtered to exactly those columns.
+ */
+@Component({
+  selector: 'df-dm-detection',
+  standalone: true,
+  imports: [
+    NgIf,
+    NgFor,
+    NgClass,
+    MatButtonModule,
+    MatDialogModule,
+    MatIconModule,
+    MatTooltipModule,
+    TranslocoPipe,
+    DfEmptyStateComponent,
+    DfSkeletonComponent,
+  ],
+  templateUrl: './df-dm-detection.component.html',
+  styleUrls: ['./df-dm-detection.component.scss'],
+})
+export class DfDmDetectionComponent implements OnInit {
+  readonly store = inject(DataMaskingStore);
+  private api = inject(DfDataMaskingApiService);
+  private dialog = inject(MatDialog);
+
+  @Output() showMatches = new EventEmitter();
+
+  builtInOpen = false;
+  readonly groups = classesByTier();
+
+  ngOnInit(): void {
+    if (!this.store.rules) this.store.loadRules();
+  }
+
+  tierOf(cls: string): string {
+    return classDef(cls)?.tier ?? 'clear';
+  }
+
+  /** The baseline that added a rule (source `baseline:`), or null. */
+  baselineOf(r: DetectionRule): string | null {
+    const id = r.source?.startsWith('baseline:')
+      ? r.source.slice('baseline:'.length)
+      : '';
+    return id ? (baselineDef(id)?.name ?? id) : null;
+  }
+
+  detectorsFor(cls: string): string[] {
+    return (
+      this.store.builtin?.classes.find(c => c.class === cls)?.detectors ?? []
+    );
+  }
+
+  /** The dialog saves through the store and stays open on a refusal. */
+  addRule(): void {
+    const data: RuleDialogData = { save: d => this.store.addRule(d) };
+    this.dialog.open(DfDmRuleDialogComponent, {
+      autoFocus: 'first-tabbable',
+      data,
+    });
+  }
+
+  /**
+   * Delete asks first and says what happens to the rule's columns: they
+   * stay masked (automatic changes never unmask). When some were masked by
+   * this rule alone, a second button unmasks those too.
+   */
+  deleteRule(rule: DetectionRule): void {
+    const t = (k: string, p?: Record) => this.store.t(k, p);
+    const { masked, ruleOnly } = this.store.footprint(rule);
+    const data: DmConfirmData = {
+      title: t('detection.deleteTitle'),
+      message: masked.length
+        ? t('detection.deleteKept', {
+            columns: this.store.plural('columns', masked.length),
+          })
+        : '',
+      confirm: t('detection.deleteConfirm'),
+      cancel: t('cancel'),
+      danger: true,
+      extra: ruleOnly.length
+        ? t('detection.deleteUnmask', { count: ruleOnly.length })
+        : undefined,
+    };
+    this.dialog
+      .open(DfDmConfirmDialogComponent, { data, autoFocus: 'dialog' })
+      .afterClosed()
+      .subscribe(choice => {
+        if (choice === 'extra') this.store.deleteRule(rule, ruleOnly);
+        else if (choice) this.store.deleteRule(rule);
+      });
+  }
+
+  openMatches(rule: DetectionRule): void {
+    this.api.testRule({ match: rule.match, pattern: rule.pattern }).subscribe({
+      next: r => {
+        this.store.showColumns('all', {
+          label: rule.pattern,
+          ids: new Set(r.matches.map(m => m.id)),
+        });
+        this.showMatches.emit();
+      },
+      error: e => this.store.fail(e),
+    });
+  }
+
+  trackRule = (_: number, r: DetectionRule) => r.id;
+}
diff --git a/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.html b/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.html
new file mode 100644
index 000000000..1fd6f2ecb
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.html
@@ -0,0 +1,110 @@
+
    {{ 'dataMasking.detection.addRule' | transloco }}
    
+
    
+  
+    {{
+      'dataMasking.detection.matchName' | transloco
+    }}
+    {{
+      'dataMasking.detection.matchValue' | transloco
+    }}
+  
+
+  
+
+  
    
+    {{
+      'dataMasking.columns.class' | transloco
+    }}
+    
+      
+        {{
+          'dataMasking.classes.' + c.id | transloco
+        }}
+      
+    
+  
    
+
+  
    
+    
+      {{ p }}
+    
+    
+      
+        {{
+          (partial
+            ? 'dataMasking.detection.matchesAtLeast'
+            : 'dataMasking.detection.matchesN'
+          ) | transloco: { columns: columnsLabel(preview.length) }
+        }}
+        
      
+          
    • 
+            {{ id }}
+          
      • 
+          
    • 
+            {{
+              'dataMasking.detection.more'
+                | transloco: { count: preview.length - 5 }
+            }}
+          
      • 
+        
    
+          
+      
+        {{
+          'dataMasking.detection.matchesN'
+            | transloco: { columns: columnsLabel(0) }
+        }}
+      
+        
+  
    
+
+
    
+  
+  
+
    
diff --git a/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.scss b/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.scss
new file mode 100644
index 000000000..ca86f35a3
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.scss
@@ -0,0 +1,124 @@
+@use '../../shared/components/df-dialog-surface' as *;
+@use '../dm-controls' as *;
+@include df-dialog-surface;
+@include dm-select;
+@include dm-segmented;
+
+// Two equal segments, so neither choice looks like the default.
+.dm-seg {
+  display: flex;
+  width: 100%;
+
+  mat-button-toggle {
+    flex: 1 1 0;
+  }
+}
+
+// Disabled primary in dark/phosphor: faint ink on a raised surface, so it
+// is visibly there but clearly unavailable.
+:host-context(.dark-theme) [mat-flat-button]:disabled {
+  --mdc-filled-button-disabled-container-color: var(--df-surface-2);
+  --mdc-filled-button-disabled-label-text-color: var(--df-text-faint);
+  border: 1px solid var(--df-border-2);
+}
+
+.form {
+  display: grid;
+  gap: var(--df-space-4);
+  min-width: 440px;
+
+  @media (max-width: 520px) {
+    min-width: 0;
+  }
+}
+
+.field {
+  display: grid;
+  gap: 6px;
+}
+
+.label {
+  color: var(--df-text-2);
+  font-size: var(--df-font-size-sm);
+  font-weight: var(--df-font-weight-medium);
+}
+
+.input {
+  background: var(--df-surface);
+  border: 1px solid var(--df-border);
+  border-radius: var(--df-radius-sm);
+  color: var(--df-text);
+  font-size: var(--df-font-size-sm);
+  height: 34px;
+  outline: none;
+  padding: 0 var(--df-space-3);
+
+  &:focus {
+    border-color: var(--df-accent);
+    box-shadow: 0 0 0 1px var(--df-accent);
+  }
+
+  &.is-invalid {
+    border-color: var(--df-danger);
+    box-shadow: 0 0 0 1px var(--df-danger);
+  }
+
+  &::placeholder {
+    color: var(--df-text-faint);
+  }
+}
+
+.mono {
+  font-family: var(--df-font-mono);
+}
+
+.dm-select {
+  height: 34px;
+  line-height: 32px;
+  width: 100%;
+}
+
+.preview {
+  background: var(--df-surface-2);
+  border: 1px solid var(--df-border-2);
+  border-radius: var(--df-radius-sm);
+  font-size: var(--df-font-size-sm);
+  // Fixed height (five ids plus "and N more"), so the dialog never jumps
+  // while the preview grows or turns invalid.
+  box-sizing: border-box;
+  height: 168px;
+  overflow: hidden;
+  padding: var(--df-space-3);
+
+  strong {
+    color: var(--df-text);
+    font-weight: var(--df-font-weight-heading);
+  }
+}
+
+.idle {
+  color: var(--df-text-muted);
+}
+
+.bad {
+  color: var(--df-danger);
+}
+
+.ids {
+  display: grid;
+  gap: 2px;
+  list-style: none;
+  margin: var(--df-space-2) 0 0;
+  padding: 0;
+
+  code {
+    color: var(--df-text-2);
+    font-family: var(--df-font-mono);
+    font-size: var(--df-font-size-xs);
+  }
+}
+
+.more {
+  color: var(--df-text-muted);
+  font-size: var(--df-font-size-xs);
+}
diff --git a/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.ts b/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.ts
new file mode 100644
index 000000000..5a843ed45
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-detection/df-dm-rule-dialog.component.ts
@@ -0,0 +1,186 @@
+import { Component, OnDestroy, OnInit, inject } from '@angular/core';
+import { NgFor, NgIf } from '@angular/common';
+import { FormsModule } from '@angular/forms';
+import { MatButtonModule } from '@angular/material/button';
+import { MatButtonToggleModule } from '@angular/material/button-toggle';
+import {
+  MAT_DIALOG_DATA,
+  MatDialogModule,
+  MatDialogRef,
+} from '@angular/material/dialog';
+import { MatSelectModule } from '@angular/material/select';
+import { TranslocoPipe } from '@ngneat/transloco';
+import {
+  Observable,
+  Subject,
+  Subscription,
+  catchError,
+  debounceTime,
+  of,
+  switchMap,
+} from 'rxjs';
+import { DfDataMaskingApiService } from '../data-masking-api.service';
+import { classesByTier } from '../data-masking.catalog';
+import { DetectionRule } from '../data-masking.types';
+import { errorStatus } from '../data-masking.store';
+import { isAppError, normalizeError } from 'src/app/shared/utilities/app-error';
+import { TranslocoService } from '@ngneat/transloco';
+
+export type RuleDraft = Pick;
+type Draft = RuleDraft;
+
+/** How the dialog saves: it stays open until this succeeds. */
+export interface RuleDialogData {
+  save: (draft: RuleDraft) => Observable;
+}
+
+/** The server's pattern length limit. */
+export const MAX_PATTERN = 500;
+
+/**
+ * Worth sending. Patterns are PCRE (atomic groups, possessive quantifiers,
+ * inline flags, (?P…)), which a JavaScript RegExp cannot judge, so
+ * the server decides validity: the live preview shows its reason.
+ */
+export function validPattern(p: string): boolean {
+  return !!p.trim() && p.length <= MAX_PATTERN;
+}
+
+/**
+ * Add rule: match on column name or value pattern, pick a class, and watch
+ * the live preview count the columns it would catch. The preview does the
+ * explaining.
+ */
+@Component({
+  selector: 'df-dm-rule-dialog',
+  standalone: true,
+  imports: [
+    NgIf,
+    NgFor,
+    FormsModule,
+    MatButtonModule,
+    MatButtonToggleModule,
+    MatDialogModule,
+    MatSelectModule,
+    TranslocoPipe,
+  ],
+  templateUrl: './df-dm-rule-dialog.component.html',
+  styleUrls: ['./df-dm-rule-dialog.component.scss'],
+})
+export class DfDmRuleDialogComponent implements OnInit, OnDestroy {
+  private api = inject(DfDataMaskingApiService);
+  private ref = inject(MatDialogRef);
+  private data = inject(MAT_DIALOG_DATA, {
+    optional: true,
+  });
+  private transloco = inject(TranslocoService);
+
+  readonly groups = classesByTier();
+  readonly maxPattern = MAX_PATTERN;
+  draft: Draft = { match: 'name', pattern: '', class: '' };
+  preview: string[] | null = null;
+  /** The server ran out of its sampling budget: the count is a floor. */
+  partial = false;
+  previewing = false;
+  saving = false;
+  /** The server's reason for refusing the rule, shown in place. */
+  saveError: string | null = null;
+  /** The server's reason the preview pattern does not compile. */
+  testError: string | null = null;
+  private test$ = new Subject();
+  private sub?: Subscription;
+
+  ngOnInit(): void {
+    this.sub = this.test$
+      .pipe(
+        debounceTime(250),
+        switchMap(() => {
+          if (!validPattern(this.draft.pattern)) return of(null);
+          this.previewing = true;
+          return this.api
+            .testRule({ match: this.draft.match, pattern: this.draft.pattern })
+            .pipe(
+              catchError(e => {
+                // A refused pattern says why, in place of a match count.
+                this.testError = this.reason(e);
+                return of(null);
+              })
+            );
+        })
+      )
+      .subscribe(r => {
+        this.previewing = false;
+        if (r) this.testError = null;
+        this.preview = r ? r.matches.map(m => m.id) : null;
+        this.partial = !!r?.partial;
+      });
+  }
+
+  ngOnDestroy(): void {
+    this.sub?.unsubscribe();
+  }
+
+  get invalid(): boolean {
+    return (
+      (!!this.draft.pattern.trim() && !validPattern(this.draft.pattern)) ||
+      !!this.testError
+    );
+  }
+
+  /** What the preview area says is wrong, if anything. */
+  get problem(): string | null {
+    if (this.saveError) return this.saveError;
+    if (this.testError) return this.testError;
+    return this.invalid
+      ? this.transloco.translate('dataMasking.detection.invalid')
+      : null;
+  }
+
+  get canSave(): boolean {
+    return (
+      validPattern(this.draft.pattern) &&
+      !this.testError &&
+      !!this.draft.class &&
+      !this.saving
+    );
+  }
+
+  private reason(e: unknown): string {
+    const err = isAppError(e) ? e : normalizeError(e);
+    return this.transloco.translate(err.fields[0]?.message || err.message);
+  }
+
+  /** "1 column" / "4 columns". */
+  columnsLabel(count: number): string {
+    return this.transloco.translate(
+      `dataMasking.plural.columns.${count === 1 ? 'one' : 'other'}`,
+      { count }
+    );
+  }
+
+  changed(): void {
+    this.saveError = null;
+    this.testError = null;
+    if (!validPattern(this.draft.pattern)) this.preview = null;
+    this.test$.next();
+  }
+
+  save(): void {
+    if (!this.canSave) return;
+    if (!this.data?.save) {
+      this.ref.close(true);
+      return;
+    }
+    this.saving = true;
+    this.saveError = null;
+    this.data.save({ ...this.draft }).subscribe({
+      next: () => this.ref.close(true),
+      error: e => {
+        this.saving = false;
+        // Not allowed (403): the store's toast carries the server's
+        // message; the rule itself is fine, so nothing shows in place.
+        this.saveError = errorStatus(e) === 403 ? null : this.reason(e);
+      },
+    });
+  }
+}
diff --git a/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.html b/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.html
new file mode 100644
index 000000000..40dca9378
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.html
@@ -0,0 +1,75 @@
+
+
+  
+
+
+
+  
    
+    
    
+      
    
+        
+        
+        {{ 'dataMasking.tiers.' + g.tier | transloco }}
+      
    
+      
    
+        
+          {{ 'dataMasking.classes.' + m.class | transloco }}
+        
+        
+          {{
+            'dataMasking.formats.' + m.format | transloco
+          }}
+          {{
+            'dataMasking.formats.' + f | transloco
+          }}
+        
+        
+          
+            {{ ex.before }}
+            
+            
+          
+          
+            {{ n }}
+            
+            
+          
+        
+      
    
+    
    
+  
    
+    
diff --git a/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.scss b/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.scss
new file mode 100644
index 000000000..933a1e47a
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.scss
@@ -0,0 +1,112 @@
+@use '../dm-tier' as *;
+@use '../dm-controls' as *;
+@include dm-tiers;
+@include dm-select;
+
+:host {
+  display: block;
+}
+
+.masks {
+  display: grid;
+  gap: var(--df-space-5);
+  max-width: 920px;
+}
+
+.tier-head {
+  align-items: center;
+  color: var(--df-text);
+  display: flex;
+  font-size: var(--df-font-size-md);
+  font-weight: var(--df-font-weight-heading);
+  gap: var(--df-space-2);
+  margin: 0 0 var(--df-space-1);
+}
+
+.dot {
+  background: var(--dm-tier-fill);
+  border-radius: 50%;
+  height: 8px;
+  width: 8px;
+}
+
+.row {
+  align-items: center;
+  border-bottom: 1px solid var(--df-border-2);
+  display: grid;
+  gap: var(--df-space-4);
+  grid-template-columns: minmax(180px, 1fr) 176px minmax(0, 1.5fr);
+  min-height: 44px;
+  padding: 0 var(--df-space-2);
+
+  &:first-of-type {
+    border-top: 1px solid var(--df-border-2);
+  }
+}
+
+.cls {
+  align-items: center;
+  color: var(--df-text);
+  display: flex;
+  font-size: var(--df-font-size-sm);
+  gap: var(--df-space-1);
+}
+
+.lock {
+  color: var(--dm-tier);
+  font-size: 14px;
+  height: 14px;
+  width: 14px;
+}
+
+// One or two examples side by side: the class sample, and for classes that
+// are often numeric, what a number comes back as.
+.examples {
+  align-items: center;
+  column-gap: var(--df-space-5);
+  display: flex;
+  flex-wrap: wrap;
+  min-width: 0;
+  row-gap: var(--df-space-1);
+}
+
+.example {
+  align-items: center;
+  display: flex;
+  gap: var(--df-space-2);
+  min-width: 0;
+}
+
+.before {
+  color: var(--df-text-muted);
+  font-family: var(--df-font-mono);
+  font-size: var(--df-font-size-xs);
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.example df-masked-value {
+  // The kept characters are the point of the example: shorten the input.
+  flex-shrink: 0;
+}
+
+.arrow {
+  color: var(--df-text-faint);
+  flex-shrink: 0;
+  font-size: 14px;
+  height: 14px;
+  width: 14px;
+}
+
+@media (max-width: 760px) {
+  .row {
+    grid-template-columns: minmax(0, 1fr) auto;
+    padding: var(--df-space-2);
+    row-gap: var(--df-space-2);
+  }
+
+  .examples {
+    grid-column: 1 / -1;
+  }
+}
diff --git a/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.ts b/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.ts
new file mode 100644
index 000000000..ceb65c5dd
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-masks/df-dm-masks.component.ts
@@ -0,0 +1,81 @@
+import { Component, OnInit, inject } from '@angular/core';
+import { NgClass, NgFor, NgIf } from '@angular/common';
+import { MatIconModule } from '@angular/material/icon';
+import { MatSelectModule } from '@angular/material/select';
+import { TranslocoPipe } from '@ngneat/transloco';
+import { DfSkeletonComponent } from 'src/app/shared/components/df-skeleton/df-skeleton.component';
+import { DfEmptyStateComponent } from 'src/app/shared/components/df-empty-state/df-empty-state.component';
+import { DfMaskedValueComponent } from '../df-masked-value/df-masked-value.component';
+import { DataMaskingStore } from '../data-masking.store';
+import { MaskFormat, TIERS, Tier } from '../data-masking.catalog';
+import { MaskSetting } from '../data-masking.types';
+import { sampleMask } from '../mask-format';
+
+/**
+ * Classes that often sit in numeric columns. A number is returned as null
+ * under every format but token, so their row shows that case too.
+ */
+const NUMERIC_SAMPLES: Record = {
+  geo_precise: '37.77493',
+  age: '41',
+  financial_personal: '84500',
+};
+
+interface TierGroup {
+  tier: Tier;
+  rows: MaskSetting[];
+}
+
+/**
+ * Masks tab: one format per class, grouped by tier. The example re-renders
+ * the moment the format changes, so the row explains itself.
+ */
+@Component({
+  selector: 'df-dm-masks',
+  standalone: true,
+  imports: [
+    NgIf,
+    NgFor,
+    NgClass,
+    MatIconModule,
+    MatSelectModule,
+    TranslocoPipe,
+    DfSkeletonComponent,
+    DfEmptyStateComponent,
+    DfMaskedValueComponent,
+  ],
+  templateUrl: './df-dm-masks.component.html',
+  styleUrls: ['./df-dm-masks.component.scss'],
+})
+export class DfDmMasksComponent implements OnInit {
+  readonly store = inject(DataMaskingStore);
+
+  ngOnInit(): void {
+    if (!this.store.masks) this.store.loadMasks();
+  }
+
+  get groups(): TierGroup[] {
+    const masks = this.store.masks ?? [];
+    return TIERS.map(tier => ({
+      tier,
+      rows: masks.filter(m => m.tier === tier),
+    })).filter(g => g.rows.length);
+  }
+
+  example(m: MaskSetting) {
+    return sampleMask(m.class, m.format);
+  }
+
+  /** The numeric case for a class that is often numeric, or null. */
+  numericExample(m: MaskSetting): string | null {
+    if (m.format === 'token' || m.format === 'null') return null;
+    return NUMERIC_SAMPLES[m.class] ?? null;
+  }
+
+  setFormat(m: MaskSetting, format: MaskFormat): void {
+    this.store.setMask(m.class, format);
+  }
+
+  trackTier = (_: number, g: TierGroup) => g.tier;
+  trackMask = (_: number, m: MaskSetting) => m.class;
+}
diff --git a/src/app/adf-data-masking/df-dm-undo-toast/df-dm-undo-toast.component.ts b/src/app/adf-data-masking/df-dm-undo-toast/df-dm-undo-toast.component.ts
new file mode 100644
index 000000000..2de0778c3
--- /dev/null
+++ b/src/app/adf-data-masking/df-dm-undo-toast/df-dm-undo-toast.component.ts
@@ -0,0 +1,71 @@
+import { Component, Inject, Injectable, Injector, inject } from '@angular/core';
+import { MatButtonModule } from '@angular/material/button';
+import {
+  MAT_SNACK_BAR_DATA,
+  MatSnackBar,
+  MatSnackBarRef,
+} from '@angular/material/snack-bar';
+import { FontAwesomeModule } from '@fortawesome/angular-fontawesome';
+import { faCheckCircle, faXmark } from '@fortawesome/free-solid-svg-icons';
+
+export interface UndoToastData {
+  message: string;
+  action: string;
+}
+
+/**
+ * The app's success toast with one action (Undo). Same surface as
+ * df-snackbar, whose stylesheet it reuses, so it reads as the same toast.
+ */
+@Component({
+  selector: 'df-dm-undo-toast',
+  standalone: true,
+  imports: [MatButtonModule, FontAwesomeModule],
+  template: `
+    
    
+      
+      {{ data.message }}
+      
+      
+    
    
+  `,
+  styleUrls: ['../../shared/components/df-snackbar/df-snackbar.component.scss'],
+})
+export class DfDmUndoToastComponent {
+  readonly icon = faCheckCircle;
+  readonly close = faXmark;
+  constructor(
+    public ref: MatSnackBarRef,
+    @Inject(MAT_SNACK_BAR_DATA) public data: UndoToastData
+  ) {}
+}
+
+/**
+ * Opens the undo toast; `onUndo` runs when its action is clicked. The snack
+ * bar is resolved when first needed (the app root provides it).
+ */
+@Injectable({ providedIn: 'root' })
+export class DmUndoToastService {
+  private injector = inject(Injector);
+
+  open(message: string, action: string, onUndo: () => void): void {
+    this.injector
+      .get(MatSnackBar)
+      .openFromComponent(DfDmUndoToastComponent, {
+        duration: 8000,
+        horizontalPosition: 'left',
+        verticalPosition: 'bottom',
+        data: { message, action },
+      })
+      .onAction()
+      .subscribe(onUndo);
+  }
+}
diff --git a/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.html b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.html
new file mode 100644
index 000000000..3320582d6
--- /dev/null
+++ b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.html
@@ -0,0 +1,66 @@
+
    
+  
      
+    
    1. 
+      {{ points(l.points) }}
+      {{ l.text }}
+    
      2. 
+    
    2. 
+      {{ total }}
+      {{
+        'dataMasking.verdict.' + detected | transloco
+      }}
+      
+        
+        {{ thresholds.review }}
+        {{ thresholds.mask }}
+      
+    
      3. 
+  
    
+
+  
    
+    
+      
    {{ 'dataMasking.ledger.type' | transloco }}
    
+      
    
+        {{ column.type }}
+      
    
+        
+    
+      
    {{ 'dataMasking.ledger.shapes' | transloco }}
    
+      
    
+        {{ s[0] }}{{ s[1] }}%
+      
    
+        
+    
+      
    {{ 'dataMasking.ledger.decision' | transloco }}
    
+      
    
+        
+        {{
+          (column.decidedBy
+            ? 'dataMasking.ledger.setBy'
+            : 'dataMasking.ledger.setByYou'
+          ) | transloco: { name: column.decidedBy, when: decidedAgo }
+        }}
+      
    
+        
+  
    
+
    
diff --git a/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.scss b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.scss
new file mode 100644
index 000000000..f6cc3bd73
--- /dev/null
+++ b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.scss
@@ -0,0 +1,190 @@
+@use '../dm-tier' as *;
+@include dm-tiers;
+
+:host {
+  display: block;
+}
+
+.ledger {
+  display: grid;
+  gap: var(--df-space-5);
+  grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr);
+  padding: var(--df-space-3) var(--df-space-4) var(--df-space-4);
+
+  @media (max-width: 720px) {
+    grid-template-columns: minmax(0, 1fr);
+    gap: var(--df-space-3);
+  }
+}
+
+.receipt {
+  list-style: none;
+  margin: 0;
+  max-width: 520px;
+  padding: 0;
+}
+
+.line {
+  align-items: baseline;
+  color: var(--df-text-2);
+  display: grid;
+  font-size: var(--df-font-size-sm);
+  gap: var(--df-space-3);
+  grid-template-columns: 40px minmax(0, 1fr);
+  line-height: var(--df-lh-base);
+  padding: 2px 0;
+
+  &.is-zero {
+    color: var(--df-text-muted);
+  }
+
+  &.is-penalty .pts {
+    color: var(--df-danger);
+  }
+}
+
+.pts {
+  color: var(--df-text);
+  font-family: var(--df-font-mono);
+  font-size: var(--df-font-size-xs);
+  font-weight: var(--df-font-weight-medium);
+  text-align: right;
+
+  .is-zero & {
+    color: var(--df-text-faint);
+  }
+}
+
+// The total: a ruled-off receipt line with the score on a thresholded bar.
+.total {
+  align-items: center;
+  border-top: 1px dashed var(--df-border);
+  grid-template-columns: 40px auto minmax(120px, 200px);
+  margin-top: var(--df-space-2);
+  padding-top: var(--df-space-2);
+
+  .pts {
+    font-size: var(--df-font-size-md);
+    font-weight: var(--df-font-weight-heading);
+  }
+}
+
+.verdict {
+  color: var(--df-text);
+  font-weight: var(--df-font-weight-medium);
+}
+
+.meter {
+  background: var(--df-surface-2);
+  border-radius: 999px;
+  box-shadow: inset 0 0 0 1px var(--df-border-2);
+  height: 6px;
+  margin-bottom: 12px;
+  position: relative;
+  top: 6px;
+}
+
+.meter-fill {
+  background: var(--df-text-faint);
+  border-radius: 999px;
+  inset: 0 auto 0 0;
+  position: absolute;
+  transition: width var(--df-duration-standard) var(--df-ease-standard);
+
+  // Same language as the chip and strip: solid tier for masked, a hollow
+  // tier outline while the column still needs review.
+  .is-masked &,
+  .is-review & {
+    background: var(--dm-tier-fill);
+  }
+
+  .is-undecided & {
+    background: var(--dm-tier-soft);
+    box-shadow: inset 0 0 0 1px var(--dm-tier);
+  }
+}
+
+.notch {
+  background: var(--df-text-muted);
+  height: 10px;
+  position: absolute;
+  top: -2px;
+  width: 1px;
+}
+
+.notch-n {
+  color: var(--df-text-faint);
+  font-size: 10px;
+  left: 50%;
+  line-height: 1;
+  position: absolute;
+  top: 12px;
+  transform: translateX(-50%);
+}
+
+.facts {
+  align-content: start;
+  display: grid;
+  font-size: var(--df-font-size-sm);
+  gap: var(--df-space-2) var(--df-space-3);
+  grid-template-columns: max-content minmax(0, 1fr);
+  margin: 0;
+
+  dt {
+    color: var(--df-text-muted);
+    padding-top: 2px;
+  }
+
+  dd {
+    color: var(--df-text-2);
+    margin: 0;
+    min-width: 0;
+  }
+
+  code {
+    color: var(--df-text-2);
+    font-family: var(--df-font-mono);
+    font-size: var(--df-font-size-xs);
+  }
+}
+
+.shapes {
+  display: flex;
+  flex-wrap: wrap;
+  gap: var(--df-space-1);
+}
+
+.shape {
+  align-items: center;
+  background: var(--df-surface-2);
+  border: 1px solid var(--df-border-2);
+  border-radius: var(--df-radius-sm);
+  display: inline-flex;
+  gap: 6px;
+  max-width: 100%;
+  padding: 1px 6px;
+
+  code {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+  }
+}
+
+.shape-pct {
+  color: var(--df-text-muted);
+  font-size: var(--df-font-size-xs);
+}
+
+.decided {
+  align-items: center;
+  display: flex;
+  gap: var(--df-space-1);
+
+  mat-icon {
+    color: var(--df-text-muted);
+    font-size: 16px;
+    height: 16px;
+    width: 16px;
+  }
+}
diff --git a/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.spec.ts b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.spec.ts
new file mode 100644
index 000000000..6397fa3e7
--- /dev/null
+++ b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.spec.ts
@@ -0,0 +1,46 @@
+import { receiptLines } from './df-evidence-ledger.component';
+import { Evidence, EvidenceLine } from '../data-masking.types';
+
+const line = (
+  points: number,
+  text: string,
+  kind: EvidenceLine['kind'] = 'name'
+): EvidenceLine => ({ points, kind, text });
+
+const ev = (total: number, lines: EvidenceLine[]): Evidence => ({
+  total,
+  thresholdMask: 70,
+  thresholdReview: 40,
+  lines,
+});
+
+describe('receiptLines', () => {
+  it('drops zero-point lines while the scored lines add up to the total', () => {
+    const lines = [
+      line(70, 'Column name "dni"'),
+      line(0, '"key id" rules out credential', 'context'),
+      line(75, '256 of 256 values are valid Spanish DNIs', 'values'),
+    ];
+    // 145 caps at 100: the receipt still adds up.
+    expect(receiptLines(ev(100, lines)).map(l => l.points)).toEqual([70, 75]);
+  });
+
+  it('keeps every line when a zero-point line explains the total', () => {
+    // Live: a strong name pulled down by values that do not match it.
+    const lines = [
+      line(70, 'Column name "client_ip"'),
+      line(0, '0 of 256 values are IPv4 addresses', 'values'),
+    ];
+    expect(receiptLines(ev(30, lines))).toEqual(lines);
+  });
+
+  it('shows the zero-point lines when nothing scored', () => {
+    const lines = [line(0, 'Key column, never classified', 'context')];
+    expect(receiptLines(ev(0, lines))).toEqual(lines);
+  });
+
+  it('copes with missing evidence', () => {
+    expect(receiptLines(null)).toEqual([]);
+    expect(receiptLines(ev(0, []))).toEqual([]);
+  });
+});
diff --git a/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.ts b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.ts
new file mode 100644
index 000000000..563bc7732
--- /dev/null
+++ b/src/app/adf-data-masking/df-evidence-ledger/df-evidence-ledger.component.ts
@@ -0,0 +1,89 @@
+import { ChangeDetectionStrategy, Component, Input } from '@angular/core';
+import { NgClass, NgFor, NgIf } from '@angular/common';
+import { MatIconModule } from '@angular/material/icon';
+import { TranslocoPipe } from '@ngneat/transloco';
+import {
+  Evidence,
+  EvidenceLine,
+  MaskedColumn,
+  Verdict,
+} from '../data-masking.types';
+import { relTime } from '../dm-time';
+
+const clampScore = (n: number) => Math.max(0, Math.min(100, n));
+
+/** The evidence lines a receipt shows (see DfEvidenceLedgerComponent.lines). */
+export function receiptLines(
+  evidence: Evidence | null | undefined
+): EvidenceLine[] {
+  const all = evidence?.lines ?? [];
+  const scored = all.filter(l => l.points !== 0);
+  if (!scored.length) return all;
+  const sum = scored.reduce((s, l) => s + l.points, 0);
+  return clampScore(sum) === clampScore(evidence?.total ?? 0) ? scored : all;
+}
+
+/**
+ * df-evidence-ledger: why a column got its verdict, as a small receipt.
+ * Each line is the points it earned; the total sits on a 0-100 bar with the
+ * review and mask thresholds notched in. Value shapes are mono chips. Raw
+ * values never appear here.
+ */
+@Component({
+  selector: 'df-evidence-ledger',
+  standalone: true,
+  changeDetection: ChangeDetectionStrategy.OnPush,
+  imports: [NgIf, NgFor, NgClass, MatIconModule, TranslocoPipe],
+  templateUrl: './df-evidence-ledger.component.html',
+  styleUrls: ['./df-evidence-ledger.component.scss'],
+})
+export class DfEvidenceLedgerComponent {
+  @Input({ required: true }) column!: MaskedColumn;
+
+  get total(): number {
+    return clampScore(this.column.evidence?.total ?? 0);
+  }
+
+  get thresholds() {
+    return {
+      review: this.column.evidence?.thresholdReview ?? 40,
+      mask: this.column.evidence?.thresholdMask ?? 70,
+    };
+  }
+
+  /** What detection alone concluded, before any admin decision. */
+  get detected(): Verdict {
+    const { review, mask } = this.thresholds;
+    return this.total >= mask
+      ? 'masked'
+      : this.total >= review
+        ? 'review'
+        : 'clear';
+  }
+
+  /** The meter wears the column's tier, like its chip and strip tick. */
+  get tierClass(): string {
+    return `dm-tier-${this.column.tier || 'clear'}`;
+  }
+
+  /**
+   * Lines that earned (or cost) points. Zero-point lines are noise in a
+   * receipt while the scored lines add up to the total (the score caps at
+   * 0-100). When they don't, a zero-point line is the reason, e.g. "0 of
+   * 256 values are IPv4 addresses" pulling a strong name down, so every
+   * line shows. All of them also show when none scored.
+   */
+  get lines() {
+    return receiptLines(this.column.evidence);
+  }
+
+  get decidedAgo(): string {
+    return relTime(this.column.decidedAt);
+  }
+
+  points(p: number): string {
+    if (p > 0) return `+${p}`;
+    if (p < 0) return `−${Math.abs(p)}`;
+    return '0';
+  }
+}
diff --git a/src/app/adf-data-masking/df-masked-value/df-masked-value.component.scss b/src/app/adf-data-masking/df-masked-value/df-masked-value.component.scss
new file mode 100644
index 000000000..e2b018a41
--- /dev/null
+++ b/src/app/adf-data-masking/df-masked-value/df-masked-value.component.scss
@@ -0,0 +1,83 @@
+@use '../dm-tier' as *;
+@include dm-tiers;
+
+:host {
+  display: inline-flex;
+  max-width: 100%;
+  min-width: 0;
+}
+
+.mv {
+  align-items: center;
+  display: inline-flex;
+  max-width: 100%;
+  min-width: 0;
+}
+
+.mv-text {
+  align-items: center;
+  color: var(--df-text-2);
+  display: inline-flex;
+  font-family: var(--df-font-mono);
+  font-size: var(--df-font-size-xs);
+  line-height: var(--df-lh-base);
+  max-width: 100%;
+  min-width: 0;
+  overflow: hidden;
+  white-space: nowrap;
+}
+
+// Kept text gives way before a bar does: in a narrow cell each plain run
+// shrinks (with an ellipsis) so the bars, the proof of masking, stay in view.
+.mv-seg {
+  flex: 0 1 auto;
+  min-width: 0;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: pre;
+}
+
+// A redaction bar: one solid, rounded slab per masked run.
+.mv-bar {
+  background: var(--dm-tier-fill);
+  border-radius: 2px;
+  display: inline-block;
+  flex: none;
+  height: 0.8em;
+  margin: 0 0.06em;
+  opacity: 0.88;
+
+  // Kept characters sit a clear gap from the bar, so "••••5911" reads as
+  // redacted-then-kept, not clipped.
+  &.before-text {
+    margin-inline-end: 0.15em;
+  }
+
+  &.after-text {
+    margin-inline-start: 0.15em;
+  }
+}
+
+.mv-token {
+  background: var(--dm-tier-soft);
+  border-radius: var(--df-radius-sm);
+  color: var(--dm-tier-ink);
+  font-family: var(--df-font-mono);
+  font-size: var(--df-font-size-xs);
+  font-weight: var(--df-font-weight-medium);
+  line-height: 1;
+  padding: 3px 6px;
+  white-space: nowrap;
+}
+
+// What the API returns for this column is null: a tinted chip like a
+// token, so it reads as masked, not as missing data.
+.mv-null {
+  background: var(--dm-tier-soft);
+  border-radius: var(--df-radius-sm);
+  color: var(--dm-tier-ink);
+  font-family: var(--df-font-mono);
+  font-size: var(--df-font-size-xs);
+  line-height: 1;
+  padding: 3px 6px;
+}
diff --git a/src/app/adf-data-masking/df-masked-value/df-masked-value.component.spec.ts b/src/app/adf-data-masking/df-masked-value/df-masked-value.component.spec.ts
new file mode 100644
index 000000000..ba377db72
--- /dev/null
+++ b/src/app/adf-data-masking/df-masked-value/df-masked-value.component.spec.ts
@@ -0,0 +1,155 @@
+import { Component } from '@angular/core';
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+import { DfMaskedValueComponent } from './df-masked-value.component';
+import { MASK_GLYPH } from '../mask-format';
+import { dmTransloco } from '../dm-testing';
+import { MaskFormat, Tier } from '../data-masking.catalog';
+
+@Component({
+  standalone: true,
+  imports: [DfMaskedValueComponent],
+  template: ``,
+})
+class HostComponent {
+  value: string | null = null;
+  format: MaskFormat | null = null;
+  tier: Tier | null = 'restricted';
+}
+
+const G = MASK_GLYPH;
+
+describe('DfMaskedValueComponent', () => {
+  let fixture: ComponentFixture;
+  let host: HostComponent;
+
+  const el = () => fixture.nativeElement as HTMLElement;
+  const render = (value: string | null, format: MaskFormat | null = null) => {
+    host.value = value;
+    host.format = format;
+    fixture.detectChanges();
+  };
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({ imports: [HostComponent, dmTransloco()] });
+    fixture = TestBed.createComponent(HostComponent);
+    host = fixture.componentInstance;
+  });
+
+  it('draws one bar per masked run and keeps plain characters', () => {
+    render(`${G}${G}${G}-${G}${G}-4821`, 'last4');
+    const bars = el().querySelectorAll('.mv-bar');
+    expect(bars.length).toBe(2);
+    expect((bars[0] as HTMLElement).style.width).toBe('1.65em');
+    expect(el().querySelector('.mv-text')?.textContent).toContain('4821');
+    expect(el().textContent).not.toContain(G);
+  });
+
+  it('speaks a single label instead of the glyphs', () => {
+    render(`${G}${G}${G}-${G}${G}-4821`, 'last4');
+    expect(el().querySelector('.mv')?.getAttribute('aria-label')).toBe(
+      'Masked, ends in 4821'
+    );
+  });
+
+  it('caps very long runs', () => {
+    render(G.repeat(40), 'redact');
+    const bar = el().querySelector('.mv-bar') as HTMLElement;
+    expect(bar.style.width).toBe('5.5em');
+    expect(el().querySelector('.mv')?.getAttribute('aria-label')).toBe(
+      'Masked'
+    );
+  });
+
+  it('renders typed tokens as a chip', () => {
+    render('EMAIL_7f3a91c2', 'token');
+    expect(el().querySelector('.mv-token')?.textContent?.trim()).toBe(
+      'EMAIL_7f3a91c2'
+    );
+    expect(el().querySelector('.mv')?.getAttribute('aria-label')).toBe(
+      'Token EMAIL_7f3a91c2'
+    );
+  });
+
+  it('renders the null format as null', () => {
+    render(null, 'null');
+    expect(el().querySelector('.mv-null')?.textContent).toBe('null');
+  });
+
+  it('a numeric column (null under redact) reads as masked null, not missing', () => {
+    render(null, 'redact');
+    const n = el().querySelector('.mv-null');
+    expect(n?.textContent).toBe('null');
+    expect(el().querySelector('.mv')?.getAttribute('aria-label')).toBe(
+      'Masked, returned as null'
+    );
+  });
+
+  it('a review column with no class or format draws a bar (the server redacts it)', () => {
+    render(null, null);
+    expect(el().querySelector('.mv-null')).toBeNull();
+    expect(el().querySelectorAll('.mv-bar').length).toBe(1);
+  });
+
+  it('tints by tier', () => {
+    render(`${G}${G}`, 'redact');
+    expect(el().querySelector('.mv')?.classList).toContain(
+      'dm-tier-restricted'
+    );
+  });
+  describe('privacy guard', () => {
+    it('draws a full bar, never the text, for a raw value', () => {
+      render('123-45-6789', 'last4');
+      expect(el().textContent).not.toContain('6789');
+      expect(el().querySelectorAll('.mv-bar').length).toBe(1);
+      expect(el().querySelector('.mv')?.getAttribute('aria-label')).toBe(
+        'Masked'
+      );
+    });
+
+    it('treats an unmasked free-text example as raw', () => {
+      render('Patient reports chest pain', 'span');
+      expect(el().textContent).not.toContain('chest');
+      expect(el().querySelectorAll('.mv-bar').length).toBe(1);
+    });
+
+    it('caps how much free text is drawn', () => {
+      render(`Call me on ${G.repeat(6)} ` + 'x'.repeat(200), 'span');
+      const text = el().querySelector('.mv-text')?.textContent ?? '';
+      expect(text.length).toBeLessThanOrEqual(48);
+    });
+
+    it('does not render a token-format value that is not a token', () => {
+      render('jane@example.com', 'token');
+      expect(el().querySelector('.mv-token')).toBeNull();
+      expect(el().textContent).not.toContain('jane');
+    });
+
+    it('allows a bare year in the year format', () => {
+      render('1984', 'year');
+      expect(el().querySelector('.mv-text')?.textContent).toContain('1984');
+    });
+
+    it('rejects a full date in the year format', () => {
+      render('1984-03-02', 'year');
+      expect(el().textContent).not.toContain('03');
+    });
+
+    it('keeps each plain run in its own shrinkable segment', () => {
+      render(`Call Jane at ${G.repeat(6)} or ${G.repeat(6)}`, 'span');
+      const segs = Array.from(el().querySelectorAll('.mv-seg')).map(
+        s => s.textContent
+      );
+      // Spaces survive (white-space: pre), so " or " reads as a word.
+      expect(segs).toEqual(['Call Jane at ', ' or ']);
+      expect(el().querySelectorAll('.mv-bar').length).toBe(2);
+    });
+
+    it('spaces the last bar from kept characters', () => {
+      render(`${G.repeat(4)}5911`, 'last4');
+      expect(el().querySelector('.mv-bar')?.classList).toContain('before-text');
+    });
+  });
+});
diff --git a/src/app/adf-data-masking/df-masked-value/df-masked-value.component.ts b/src/app/adf-data-masking/df-masked-value/df-masked-value.component.ts
new file mode 100644
index 000000000..f89ee4c99
--- /dev/null
+++ b/src/app/adf-data-masking/df-masked-value/df-masked-value.component.ts
@@ -0,0 +1,162 @@
+import {
+  ChangeDetectionStrategy,
+  Component,
+  Input,
+  OnChanges,
+  inject,
+} from '@angular/core';
+import { NgClass, NgFor, NgIf } from '@angular/common';
+import { TranslocoService } from '@ngneat/transloco';
+import { MaskFormat, Tier } from '../data-masking.catalog';
+import {
+  MASK_GLYPH,
+  MaskSegment,
+  TOKEN_RE,
+  segmentMasked,
+} from '../mask-format';
+
+/** Longest bar, in glyphs: long runs read as "a lot", not a ruler. */
+const MAX_BAR = 10;
+/** Most characters of free text (span format) ever drawn. */
+export const SPAN_MAX_CHARS = 48;
+/** What the guard draws instead of an unmasked value. */
+const FULL_BAR = MASK_GLYPH.repeat(8);
+
+/**
+ * Privacy guard. A masked/review value must visibly carry its mask; this
+ * never trusts the server to have done it. Returns what is safe to draw:
+ * the value itself, a capped span text, or a full redaction bar.
+ */
+export function safeMasked(
+  value: string,
+  format: MaskFormat | null | undefined
+): string {
+  if (value.includes(MASK_GLYPH)) {
+    return format === 'span' ? capSpan(value) : value;
+  }
+  if (TOKEN_RE.test(value)) return value;
+  // Year only legitimately keeps every character it shows: a bare year.
+  if (format === 'year' && /^\d{2,4}$/.test(value.trim())) return value;
+  // Anything else without a mask in it is a raw value: draw a bar.
+  return FULL_BAR;
+}
+
+function capSpan(value: string): string {
+  return value.length > SPAN_MAX_CHARS
+    ? `${value.slice(0, SPAN_MAX_CHARS - 1)}…`
+    : value;
+}
+
+/**
+ * df-masked-value: a masked value drawn like a redacted document. Runs of
+ * the mask glyph become solid tier-tinted bars; kept characters stay plain;
+ * typed tokens render as a mono chip, and a null (the null format, or a
+ * numeric value under any other format, which the API returns as null) as a
+ * tinted `null` chip. A missing value with no format (a review column with
+ * no class, which the server redacts) draws a bar. The visible marks are
+ * aria-hidden and the host carries one spoken label ("Masked, ends in 4821").
+ */
+@Component({
+  selector: 'df-masked-value',
+  standalone: true,
+  changeDetection: ChangeDetectionStrategy.OnPush,
+  imports: [NgIf, NgFor, NgClass],
+  template: `
+    
+      
+        
+      
+      
+        
+      
+      
+        
+      
+    
+  `,
+  styleUrls: ['./df-masked-value.component.scss'],
+})
+export class DfMaskedValueComponent implements OnChanges {
+  private transloco = inject(TranslocoService);
+
+  @Input() value: string | null | undefined = null;
+  @Input() tier: Tier | null = null;
+  @Input() format: MaskFormat | null | undefined = null;
+
+  segments: MaskSegment[] = [];
+  token = false;
+  isNull = false;
+  label = '';
+
+  /** The guarded value actually drawn. */
+  shown = '';
+
+  /** A null that stands for a non-string value (format other than null). */
+  private nullValue = false;
+
+  ngOnChanges(): void {
+    const raw = this.value;
+    const missing = raw === null || raw === undefined;
+    this.isNull = this.format === 'null' || (missing && !!this.format);
+    this.nullValue = this.isNull && this.format !== 'null';
+    const v = this.isNull
+      ? ''
+      : missing
+        ? FULL_BAR
+        : safeMasked(String(raw), this.format);
+    this.token = !this.isNull && TOKEN_RE.test(v);
+    this.shown = v;
+    this.segments = this.isNull || this.token ? [] : segmentMasked(v);
+    this.label = this.spoken();
+  }
+
+  barWidth(s: MaskSegment): number {
+    return Math.round(Math.min(s.length, MAX_BAR) * 55) / 100;
+  }
+
+  trackSeg = (i: number) => i;
+
+  /** Is the segment at `i` (if any) kept text? Spaces bars from it. */
+  isText(i: number): boolean {
+    return this.segments[i]?.kind === 'text';
+  }
+
+  private spoken(): string {
+    const t = (k: string, p?: Record) =>
+      this.transloco.translate(`dataMasking.mv.${k}`, p);
+    if (this.isNull) return t(this.nullValue ? 'nullValue' : 'null');
+    if (this.token) return t('token', { token: this.shown });
+    const bars = this.segments.filter(s => s.kind === 'bar').length;
+    // Never speak a value that is not visibly masked.
+    if (!bars) return t('masked');
+    const last = this.segments[this.segments.length - 1];
+    if (last.kind === 'text') {
+      const tail = last.text.replace(/[^A-Za-z0-9]/g, '');
+      if (tail) return t('endsIn', { tail });
+    }
+    const first = this.segments[0];
+    if (first.kind === 'text' && /[A-Za-z0-9]/.test(first.text)) {
+      return t('startsWith', { head: first.text.trim() });
+    }
+    return t('masked');
+  }
+}
diff --git a/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.scss b/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.scss
new file mode 100644
index 000000000..2df67226c
--- /dev/null
+++ b/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.scss
@@ -0,0 +1,43 @@
+@use '../dm-tier' as *;
+
+// The strip fills the fixed slot its parent gives it; ticks are 4px at
+// most and squeeze to 1px, so wide and narrow tables line up down the rail.
+:host {
+  display: flex;
+  min-width: 0;
+  width: 100%;
+}
+
+.strip {
+  align-items: stretch;
+  display: flex;
+  gap: 1px;
+  height: 12px;
+  max-width: 100%;
+  min-width: 0;
+  // Ticks never spill into the badge beside the slot.
+  overflow: hidden;
+}
+
+.tick {
+  background: var(--df-mask-clear);
+  border-radius: 1px;
+  flex: 0 1 4px;
+  // MAX_TICKS (24) ticks and their gaps fit the narrowest slot (48px).
+  min-width: 1px;
+}
+
+@include dm-tiers;
+
+.tick.dm-tier-restricted,
+.tick.dm-tier-identity,
+.tick.dm-tier-quasi,
+.tick.dm-tier-free_text {
+  background: var(--dm-tier-fill);
+}
+
+// Needs review: hollow, so "undecided" reads at a glance.
+.tick.is-review {
+  background: transparent;
+  box-shadow: inset 0 0 0 1px var(--dm-tier);
+}
diff --git a/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.spec.ts b/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.spec.ts
new file mode 100644
index 000000000..0b81fb99e
--- /dev/null
+++ b/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.spec.ts
@@ -0,0 +1,44 @@
+import { MAX_TICKS, compressStrip } from './df-sensitivity-strip.component';
+import { StripMark } from '../data-masking.types';
+
+const item = (mark: StripMark, i: number) => ({
+  column: `c${i}`,
+  mark,
+  tier: mark === 'review' ? ('identity' as const) : null,
+});
+
+describe('compressStrip', () => {
+  it('keeps narrow tables as they are', () => {
+    const items = Array.from({ length: 10 }, (_, i) => item('clear', i));
+    expect(compressStrip(items)).toBe(items);
+  });
+
+  it('squeezes clear runs and keeps every sensitive tick when that fits', () => {
+    const items = Array.from({ length: 200 }, (_, i) =>
+      item(i % 20 === 0 ? 'restricted' : 'clear', i)
+    );
+    const out = compressStrip(items);
+    expect(out.length).toBeLessThanOrEqual(MAX_TICKS);
+    expect(out.filter(s => s.mark === 'restricted').length).toBe(10);
+  });
+
+  it('never draws more than MAX_TICKS, even when most columns are sensitive', () => {
+    // 40 sensitive columns used to give 40+ ticks and cover the badge.
+    const items = Array.from({ length: 60 }, (_, i) =>
+      item(i % 3 === 0 ? 'clear' : i % 2 ? 'identity' : 'quasi', i)
+    );
+    const out = compressStrip(items);
+    expect(out.length).toBe(MAX_TICKS);
+    // Sensitive data never disappears into grey.
+    expect(out.every(s => s.mark !== 'clear')).toBe(true);
+  });
+
+  it('a shared tick shows the strictest column it stands for', () => {
+    const items = Array.from({ length: 48 }, (_, i) =>
+      item(i === 47 ? 'restricted' : 'quasi', i)
+    );
+    const out = compressStrip(items);
+    expect(out.length).toBe(MAX_TICKS);
+    expect(out[MAX_TICKS - 1].mark).toBe('restricted');
+  });
+});
diff --git a/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.ts b/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.ts
new file mode 100644
index 000000000..e0bd009ae
--- /dev/null
+++ b/src/app/adf-data-masking/df-sensitivity-strip/df-sensitivity-strip.component.ts
@@ -0,0 +1,149 @@
+import {
+  ChangeDetectionStrategy,
+  Component,
+  Input,
+  OnChanges,
+} from '@angular/core';
+import { NgFor } from '@angular/common';
+import { MatTooltipModule } from '@angular/material/tooltip';
+import { StripMark, TableSummary } from '../data-masking.types';
+
+/** Most ticks a strip draws; wider tables squeeze their clear runs. */
+export const MAX_TICKS = 24;
+/** Sensitive names listed in the tooltip before "+N". */
+const TIP_NAMES = 12;
+
+interface Tick {
+  cls: string;
+}
+
+type StripItem = TableSummary['strip'][number];
+
+/**
+ * df-sensitivity-strip: one tick per column, coloured by tier, so a table's
+ * sensitive columns are visible without reading. Clear columns are faint,
+ * review columns hollow. It sits in a fixed slot and never draws more than
+ * MAX_TICKS ticks: past that, clear runs are squeezed first, and when the
+ * sensitive columns alone are too many, neighbours share a tick that
+ * shows the strictest of them, so sensitive data never disappears into
+ * grey. One tooltip on the whole strip names the sensitive columns.
+ */
+@Component({
+  selector: 'df-sensitivity-strip',
+  standalone: true,
+  changeDetection: ChangeDetectionStrategy.OnPush,
+  imports: [NgFor, MatTooltipModule],
+  template: `
+    
+      
+    
+  `,
+  styleUrls: ['./df-sensitivity-strip.component.scss'],
+})
+export class DfSensitivityStripComponent implements OnChanges {
+  @Input() strip: TableSummary['strip'] = [];
+  /** Spoken summary, already translated. */
+  @Input() label = '';
+
+  ticks: Tick[] = [];
+  tip = '';
+
+  ngOnChanges(): void {
+    const items = this.strip ?? [];
+    this.ticks = compressStrip(items).map(s => ({
+      cls: `tick ${tickClass(s.mark, s.tier)}`,
+    }));
+    const names = items.filter(s => s.mark !== 'clear').map(s => s.column);
+    const extra = names.length - TIP_NAMES;
+    this.tip =
+      names.slice(0, TIP_NAMES).join(', ') + (extra > 0 ? ` +${extra}` : '');
+  }
+
+  trackTick = (i: number) => i;
+}
+
+/** Strictness for sharing a tick: the strictest neighbour is drawn. */
+function rank(s: StripItem): number {
+  const tier = s.mark === 'review' ? s.tier : s.mark;
+  const base =
+    tier === 'restricted'
+      ? 8
+      : tier === 'identity'
+        ? 6
+        : tier === 'free_text'
+          ? 4
+          : tier === 'quasi'
+            ? 2
+            : 0;
+  // A decided mask outranks an undecided one of the same tier.
+  return s.mark === 'review' ? Math.max(base - 1, 1) : base;
+}
+
+/**
+ * At most MAX_TICKS ticks. Keep every sensitive mark and shrink runs of
+ * clear marks when that fits (a clear run never vanishes entirely);
+ * otherwise split the columns into MAX_TICKS even buckets, each drawn as
+ * its strictest mark.
+ */
+export function compressStrip(items: StripItem[]): StripItem[] {
+  if (items.length <= MAX_TICKS) return items;
+  const sensitive = items.filter(s => s.mark !== 'clear').length;
+  const clear = items.length - sensitive;
+  const runs: StripItem[][] = [];
+  for (const s of items) {
+    const last = runs[runs.length - 1];
+    if (last && s.mark === 'clear' && last[0].mark === 'clear') last.push(s);
+    else runs.push([s]);
+  }
+  const clearRuns = runs.filter(r => r[0].mark === 'clear').length;
+  if (sensitive + clearRuns <= MAX_TICKS) {
+    const budget = MAX_TICKS - sensitive;
+    const out: StripItem[][] = runs.map(r =>
+      r[0].mark !== 'clear'
+        ? r
+        : r.slice(0, Math.max(1, Math.round((r.length / clear) * budget)))
+    );
+    // Rounding can overshoot: trim the longest clear runs back to the cap.
+    let total = out.reduce((n, r) => n + r.length, 0);
+    while (total > MAX_TICKS) {
+      let longest = -1;
+      out.forEach((r, i) => {
+        if (
+          r[0].mark === 'clear' &&
+          r.length > 1 &&
+          (longest < 0 || r.length > out[longest].length)
+        ) {
+          longest = i;
+        }
+      });
+      if (longest < 0) break;
+      out[longest] = out[longest].slice(0, -1);
+      total--;
+    }
+    if (total <= MAX_TICKS) return out.flat();
+  }
+  const res: StripItem[] = [];
+  for (let b = 0; b < MAX_TICKS; b++) {
+    const start = Math.floor((b * items.length) / MAX_TICKS);
+    const end = Math.floor(((b + 1) * items.length) / MAX_TICKS);
+    let best = items[start];
+    for (let i = start + 1; i < end; i++) {
+      if (rank(items[i]) > rank(best)) best = items[i];
+    }
+    res.push(best);
+  }
+  return res;
+}
+
+export function tickClass(mark: StripMark, tier?: string | null): string {
+  return mark === 'review'
+    ? `is-review dm-tier-${tier || 'clear'}`
+    : `dm-tier-${mark}`;
+}
diff --git a/src/app/adf-data-masking/dm-testing.ts b/src/app/adf-data-masking/dm-testing.ts
new file mode 100644
index 000000000..81c3863c0
--- /dev/null
+++ b/src/app/adf-data-masking/dm-testing.ts
@@ -0,0 +1,39 @@
+/** Shared Jest helpers for the Data Masking specs. */
+import { Translation, TranslocoTestingModule } from '@ngneat/transloco';
+import { MaskedColumn } from './data-masking.types';
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const DM: Translation = require('../../assets/i18n/dataMasking/en.json');
+
+export function dmTransloco() {
+  return TranslocoTestingModule.forRoot({
+    langs: { en: { dataMasking: DM } },
+    translocoConfig: { availableLangs: ['en'], defaultLang: 'en' },
+    preloadLangs: true,
+  });
+}
+
+export function column(over: Partial = {}): MaskedColumn {
+  return {
+    id: 'crm.users.ssn',
+    service: 'crm',
+    table: 'users',
+    column: 'ssn',
+    type: 'varchar(11)',
+    verdict: 'masked',
+    class: 'national_id',
+    tier: 'restricted',
+    source: 'detected',
+    confidence: 100,
+    example: '•••-••-4821',
+    format: 'last4',
+    evidence: {
+      total: 100,
+      thresholdMask: 70,
+      thresholdReview: 40,
+      lines: [{ points: 70, kind: 'name', text: 'Column name "ssn"' }],
+      shapes: [['999-99-9999', 100]],
+    },
+    ...over,
+  };
+}
diff --git a/src/app/adf-data-masking/dm-time.spec.ts b/src/app/adf-data-masking/dm-time.spec.ts
new file mode 100644
index 000000000..c5178e844
--- /dev/null
+++ b/src/app/adf-data-masking/dm-time.spec.ts
@@ -0,0 +1,23 @@
+import { relTime } from './dm-time';
+
+describe('relTime', () => {
+  const now = new Date(2026, 9, 29, 0, 30).getTime();
+
+  it('counts calendar days, matching the day headings', () => {
+    // 26 hours earlier is two calendar days back, not "yesterday".
+    const twoDaysBack = new Date(2026, 9, 27, 22, 30).toISOString();
+    expect(relTime(twoDaysBack, now)).toBe('2 days ago');
+    expect(relTime(new Date(2026, 9, 28, 0, 0).toISOString(), now)).toBe(
+      'yesterday'
+    );
+  });
+
+  it('keeps hours and minutes inside a day', () => {
+    expect(relTime(new Date(2026, 9, 29, 0, 16).toISOString(), now)).toBe(
+      '14 minutes ago'
+    );
+    expect(relTime(new Date(2026, 9, 28, 20, 30).toISOString(), now)).toBe(
+      '4 hours ago'
+    );
+  });
+});
diff --git a/src/app/adf-data-masking/dm-time.ts b/src/app/adf-data-masking/dm-time.ts
new file mode 100644
index 000000000..aae966ae5
--- /dev/null
+++ b/src/app/adf-data-masking/dm-time.ts
@@ -0,0 +1,39 @@
+/** Relative time ("2 days ago", "in 7 hours") with Intl, no deps. */
+const UNITS: [Intl.RelativeTimeFormatUnit, number][] = [
+  ['year', 365 * 86_400_000],
+  ['month', 30 * 86_400_000],
+  ['week', 7 * 86_400_000],
+  ['day', 86_400_000],
+  ['hour', 3_600_000],
+  ['minute', 60_000],
+];
+
+export function relTime(iso: string | null | undefined, now = Date.now()) {
+  if (!iso) return '';
+  const t = Date.parse(iso);
+  if (Number.isNaN(t)) return '';
+  const diff = t - now;
+  const rtf = new Intl.RelativeTimeFormat('en', { numeric: 'auto' });
+  for (const [unit, ms] of UNITS) {
+    if (Math.abs(diff) < ms) continue;
+    // Days count calendar days, so a row never says "yesterday" under an
+    // older day heading.
+    if (unit === 'day') return rtf.format(calendarDays(t, now), unit);
+    return rtf.format(Math.round(diff / ms), unit);
+  }
+  return rtf.format(0, 'minute').replace('this minute', 'just now');
+}
+
+function calendarDays(t: number, now: number): number {
+  const midnight = (ms: number) => new Date(ms).setHours(0, 0, 0, 0);
+  return Math.round((midnight(t) - midnight(now)) / 86_400_000);
+}
+
+/** Short clock or date for an expiry: "6:42 AM" within a day, else "Nov 2". */
+export function shortWhen(iso: string, now = Date.now()): string {
+  const d = new Date(iso);
+  const soon = Math.abs(d.getTime() - now) < 86_400_000;
+  return soon
+    ? d.toLocaleTimeString('en', { hour: 'numeric', minute: '2-digit' })
+    : d.toLocaleDateString('en', { month: 'short', day: 'numeric' });
+}
diff --git a/src/app/adf-data-masking/mask-format.spec.ts b/src/app/adf-data-masking/mask-format.spec.ts
new file mode 100644
index 000000000..f461bafc7
--- /dev/null
+++ b/src/app/adf-data-masking/mask-format.spec.ts
@@ -0,0 +1,71 @@
+import {
+  MASK_GLYPH,
+  TOKEN_RE,
+  applyMask,
+  sampleMask,
+  segmentMasked,
+} from './mask-format';
+
+const G = MASK_GLYPH;
+
+describe('mask-format', () => {
+  it('last4 keeps separators and the last four characters', () => {
+    expect(applyMask('123-45-6789', 'last4', 'national_id')).toBe(
+      `${G}${G}${G}-${G}${G}-6789`
+    );
+  });
+
+  it('first letter keeps the shape of an email', () => {
+    expect(applyMask('jane@example.com', 'initial', 'email')).toBe(
+      `j${G}${G}${G}@${G.repeat(7)}.com`
+    );
+  });
+
+  it('year only keeps the year of a date', () => {
+    expect(applyMask('1984-03-12', 'year', 'date_of_birth')).toBe(
+      `1984-${G}${G}-${G}${G}`
+    );
+  });
+
+  it('token is stable and typed', () => {
+    const a = applyMask('jane@example.com', 'token', 'email');
+    expect(a).toMatch(/^EMAIL_[0-9a-f]{8}$/);
+    expect(applyMask('jane@example.com', 'token', 'email')).toBe(a);
+  });
+
+  it('null returns null and redact hides length', () => {
+    expect(applyMask('x', 'null', 'email')).toBeNull();
+    expect(applyMask('a much longer secret', 'redact', 'credential')).toBe(
+      G.repeat(8)
+    );
+  });
+
+  it('span masks only personal data inside free text', () => {
+    const out = sampleMask('free_text_pii', 'span').after as string;
+    expect(out.startsWith('Call Jane at ')).toBe(true);
+    expect(out).not.toContain('@');
+    expect(out).not.toContain('555');
+  });
+
+  it('uses the server mask glyph (U+2022)', () => {
+    expect(MASK_GLYPH).toBe('\u2022');
+    expect(applyMask('123-45-6789', 'last4', 'national_id')).toBe(
+      '\u2022\u2022\u2022-\u2022\u2022-6789'
+    );
+  });
+
+  it('accepts server tokens (PREFIX_ + 8 hex) and nothing that looks raw', () => {
+    expect(TOKEN_RE.test('EMAIL_7f3a91c2')).toBe(true);
+    expect(TOKEN_RE.test('NID_0a1b')).toBe(true);
+    expect(TOKEN_RE.test('EMAIL_7F3A91C2')).toBe(false);
+    expect(TOKEN_RE.test('jane@example.com')).toBe(false);
+    expect(TOKEN_RE.test('EMAIL_' + 'a'.repeat(17))).toBe(false);
+  });
+
+  it('segments runs of the glyph into bars', () => {
+    expect(segmentMasked(`${G}${G}-4821`)).toEqual([
+      { kind: 'bar', text: `${G}${G}`, length: 2 },
+      { kind: 'text', text: '-4821', length: 5 },
+    ]);
+  });
+});
diff --git a/src/app/adf-data-masking/mask-format.ts b/src/app/adf-data-masking/mask-format.ts
new file mode 100644
index 000000000..473c5142d
--- /dev/null
+++ b/src/app/adf-data-masking/mask-format.ts
@@ -0,0 +1,178 @@
+/**
+ * Renders a value the way a mask format would return it. Masked characters
+ * become MASK_GLYPH runs, which  paints as redaction bars.
+ * Pure TypeScript: shared by the UI (Masks tab examples), the Playwright
+ * mock and the fixture generator.
+ */
+import { MaskFormat, classDef } from './data-masking.catalog';
+
+/**
+ * U+2022 BULLET: the mask character in every masked value the server
+ * returns (backend contract, section 2). The UI paints runs of it as bars.
+ */
+export const MASK_GLYPH = '\u2022';
+
+const REDACT_RUN = 8;
+const SPAN_RUN = 6;
+
+const SPAN_PATTERNS: RegExp[] = [
+  /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, // email
+  /\b(?:\d[ -]?){13,19}\b/g, // card-like digit runs
+  /\b\d{3}-\d{2}-\d{4}\b/g, // US SSN
+  /(?:\+\d{1,3}[ .-]?)?\(?\d{3}\)?[ .-]?\d{3}[ .-]?\d{4}\b/g, // phone
+];
+
+const run = (n: number) => MASK_GLYPH.repeat(Math.max(1, n));
+
+/**
+ * FNV-1a, 8 lowercase hex chars, the same shape as the server's tokens
+ * (first 8 hex of a keyed HMAC-SHA256). Stable across runs; not a secret:
+ * it only drives synthetic examples and the Playwright mock.
+ */
+export function shortHash(value: string): string {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < value.length; i++) {
+    h ^= value.charCodeAt(i);
+    h = Math.imul(h, 0x01000193) >>> 0;
+  }
+  return (h >>> 0).toString(16).padStart(8, '0');
+}
+
+function maskAllButLast(value: string, keep: number): string {
+  const alnum = value.replace(/[^A-Za-z0-9]/g, '').length;
+  let toMask = Math.max(0, alnum - keep);
+  if (alnum <= keep) return run(REDACT_RUN);
+  let out = '';
+  for (const ch of value) {
+    if (/[A-Za-z0-9]/.test(ch) && toMask > 0) {
+      out += MASK_GLYPH;
+      toMask--;
+    } else {
+      out += ch;
+    }
+  }
+  return out;
+}
+
+function initials(value: string): string {
+  const at = value.indexOf('@');
+  if (at > 0) {
+    const local = value.slice(0, at);
+    const domain = value.slice(at + 1);
+    const dot = domain.lastIndexOf('.');
+    const host = dot > 0 ? domain.slice(0, dot) : domain;
+    const tld = dot > 0 ? domain.slice(dot) : '';
+    return `${local[0]}${run(local.length - 1)}@${run(host.length)}${tld}`;
+  }
+  return value
+    .split(/(\s+)/)
+    .map(part =>
+      /^\s+$/.test(part) || part.length === 0
+        ? part
+        : `${part[0]}${part.length > 1 ? run(part.length - 1) : ''}`
+    )
+    .join('');
+}
+
+function yearOnly(value: string): string {
+  const m = value.match(/\b(19|20)\d{2}\b/);
+  if (m && m.index !== undefined) {
+    const before = value.slice(0, m.index).replace(/[0-9]/g, MASK_GLYPH);
+    const after = value
+      .slice(m.index + 4)
+      .replace(/[0-9]/g, MASK_GLYPH)
+      .replace(/T.*$/, '');
+    return `${before}${m[0]}${after}`;
+  }
+  // MM/YY: keep the trailing group (the year).
+  const parts = value.split(/([/-])/);
+  if (parts.length >= 3) {
+    return parts
+      .map((p, i) =>
+        i === parts.length - 1 || /^[/-]$/.test(p) ? p : run(p.length)
+      )
+      .join('');
+  }
+  return run(REDACT_RUN);
+}
+
+/** Replace detected personal-data spans inside free text. */
+export function maskSpans(text: string): string {
+  let out = text;
+  for (const re of SPAN_PATTERNS) {
+    out = out.replace(re, () => run(SPAN_RUN));
+  }
+  return out;
+}
+
+export function tokenFor(classId: string, value: string): string {
+  const prefix = classDef(classId)?.token ?? 'VAL';
+  return `${prefix}_${shortHash(`${classId}:${value}`)}`;
+}
+
+/**
+ * Masked rendering of `value` for `format`. Returns null for the null
+ * format. `classId` only matters for token prefixes.
+ */
+export function applyMask(
+  value: string,
+  format: MaskFormat,
+  classId: string
+): string | null {
+  const v = String(value ?? '');
+  switch (format) {
+    case 'null':
+      return null;
+    case 'token':
+      return tokenFor(classId, v);
+    case 'last4':
+      return maskAllButLast(v, 4);
+    case 'initial':
+      return v ? initials(v) : run(REDACT_RUN);
+    case 'year':
+      return yearOnly(v);
+    case 'span':
+      return maskSpans(v);
+    case 'redact':
+    default:
+      return run(REDACT_RUN);
+  }
+}
+
+/** The Masks tab example: the class's synthetic sample through `format`. */
+export function sampleMask(
+  classId: string,
+  format: MaskFormat
+): { before: string; after: string | null } {
+  const def = classDef(classId);
+  const before = def?.sample ?? '';
+  return { before, after: applyMask(before, format, classId) };
+}
+
+export interface MaskSegment {
+  kind: 'text' | 'bar';
+  text: string;
+  /** Glyph count for bars. */
+  length: number;
+}
+
+/** Split a masked string into plain text and bar runs. */
+export function segmentMasked(masked: string): MaskSegment[] {
+  const segs: MaskSegment[] = [];
+  const re = new RegExp(`${MASK_GLYPH}+|[^${MASK_GLYPH}]+`, 'g');
+  for (const m of masked.match(re) ?? []) {
+    if (m[0] === MASK_GLYPH) {
+      segs.push({ kind: 'bar', text: m, length: m.length });
+    } else {
+      segs.push({ kind: 'text', text: m, length: m.length });
+    }
+  }
+  return segs;
+}
+
+/**
+ * A typed token: `_`. The server emits 8 lowercase hex chars;
+ * 4-16 are accepted so a future key/prefix change never renders a token as
+ * a raw value (the masked-value guard would draw a full bar instead).
+ */
+export const TOKEN_RE = /^[A-Z][A-Z0-9]*_[0-9a-f]{4,16}$/;
diff --git a/src/app/adf-section-overviews/df-security-overview.component.spec.ts b/src/app/adf-section-overviews/df-security-overview.component.spec.ts
new file mode 100644
index 000000000..5fa9c8196
--- /dev/null
+++ b/src/app/adf-section-overviews/df-security-overview.component.spec.ts
@@ -0,0 +1,128 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+import { provideRouter } from '@angular/router';
+import { Translation, TranslocoTestingModule } from '@ngneat/transloco';
+import { of, throwError } from 'rxjs';
+import { DfSecurityOverviewComponent } from './df-security-overview.component';
+import { SERVICE_TYPE_SERVICE_TOKEN } from '../shared/constants/tokens';
+import { DfPaywallService } from '../shared/services/df-paywall.service';
+import { DfSystemConfigDataService } from '../shared/services/df-system-config-data.service';
+import { DfDataMaskingApiService } from '../adf-data-masking/data-masking-api.service';
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const ROOT: Translation = require('../../assets/i18n/en.json');
+
+/** Security overview: Data Masking's status on its card (UC-14). */
+describe('DfSecurityOverviewComponent masking status', () => {
+  let fixture: ComponentFixture;
+  let overview: jest.Mock;
+  let activatePaywall: jest.Mock;
+
+  function create(license: string) {
+    TestBed.configureTestingModule({
+      imports: [
+        DfSecurityOverviewComponent,
+        TranslocoTestingModule.forRoot({
+          langs: { en: ROOT },
+          translocoConfig: { availableLangs: ['en'], defaultLang: 'en' },
+          preloadLangs: true,
+        }),
+      ],
+      providers: [
+        provideRouter([]),
+        {
+          provide: SERVICE_TYPE_SERVICE_TOKEN,
+          useValue: { getAll: () => of({ resource: [] }) },
+        },
+        {
+          provide: DfSystemConfigDataService,
+          useValue: { environment: { platform: { license } } },
+        },
+        {
+          provide: DfPaywallService,
+          useValue: {
+            activatePaywall,
+            isFeatureLocked: (route: string, l: string) =>
+              l !== 'GOLD' && route.includes('data-masking'),
+          },
+        },
+        { provide: DfDataMaskingApiService, useValue: { overview } },
+      ],
+    });
+    fixture = TestBed.createComponent(DfSecurityOverviewComponent);
+    fixture.detectChanges();
+  }
+
+  const maskingCard = () =>
+    fixture.componentInstance.groups[0].cards.find(c =>
+      c.route.endsWith('/data-masking')
+    )!;
+  const metaText = () =>
+    Array.from(
+      (fixture.nativeElement as HTMLElement).querySelectorAll('a.section-card')
+    )
+      .find(a => a.getAttribute('aria-label') === 'Data Masking')
+      ?.querySelector('.section-card__meta')
+      ?.textContent?.trim();
+
+  const ov = (
+    mode: string,
+    counts = { masked: 217, review: 29, clear: 3 }
+  ) => ({
+    mode,
+    scan: { status: 'idle', finishedAt: '2026-10-29T06:00:00Z', columns: 249 },
+    counts,
+    you: { canGrant: true, seesRealValues: false },
+  });
+
+  beforeEach(() => {
+    overview = jest.fn(() => of(ov('preview')));
+    activatePaywall = jest.fn(() => of(false));
+  });
+
+  it('shows the mode and the review queue', () => {
+    create('GOLD');
+    expect(overview).toHaveBeenCalledWith(true);
+    expect(maskingCard().meta).toBe('Preview · 29 to review');
+    expect(metaText()).toBe('Preview · 29 to review');
+  });
+
+  it('On shows masked and to-review counts, disjoint', () => {
+    overview.mockReturnValue(of(ov('on')));
+    create('GOLD');
+    expect(maskingCard().meta).toBe('On · 217 masked · 29 to review');
+  });
+
+  it('says when nothing has been scanned yet', () => {
+    overview.mockReturnValue(
+      of({
+        ...ov('off'),
+        scan: { status: 'idle', finishedAt: null, columns: 0 },
+      })
+    );
+    create('GOLD');
+    expect(maskingCard().meta).toBe('Not scanned');
+  });
+
+  it('a GOLD instance without the package is paywalled: no request, no status line', () => {
+    activatePaywall.mockReturnValue(of(true));
+    create('GOLD');
+    expect(activatePaywall).toHaveBeenCalledWith('data_masking');
+    expect(overview).not.toHaveBeenCalled();
+    expect(maskingCard().meta).toBeUndefined();
+    expect(metaText()).toBeUndefined();
+  });
+
+  it('open source looks as before: no request, no status line', () => {
+    create('OPEN SOURCE');
+    expect(activatePaywall).not.toHaveBeenCalled();
+    expect(overview).not.toHaveBeenCalled();
+    expect(maskingCard().meta).toBeUndefined();
+    expect(metaText()).toBeUndefined();
+  });
+
+  it('a failed read leaves the card as before', () => {
+    overview.mockReturnValue(throwError(() => ({ status: 403 })));
+    create('GOLD');
+    expect(maskingCard().meta).toBeUndefined();
+  });
+});
diff --git a/src/app/adf-section-overviews/df-security-overview.component.ts b/src/app/adf-section-overviews/df-security-overview.component.ts
index 4c6e2e68e..c669487cc 100644
--- a/src/app/adf-section-overviews/df-security-overview.component.ts
+++ b/src/app/adf-section-overviews/df-security-overview.component.ts
@@ -1,7 +1,7 @@
 import { CommonModule } from '@angular/common';
 import { Component, OnInit, inject } from '@angular/core';
 import { TranslocoService } from '@ngneat/transloco';
-import { catchError, map, of } from 'rxjs';
+import { catchError, map, of, switchMap, take } from 'rxjs';
 import {
   DfSectionLandingComponent,
   SectionLandingAction,
@@ -13,6 +13,10 @@ import { SERVICE_TYPE_SERVICE_TOKEN } from '../shared/constants/tokens';
 import { GenericListResponse } from '../shared/types/generic-http';
 import { ROUTES } from '../shared/types/routes';
 import { ServiceType } from '../shared/types/service';
+import { DfPaywallService } from '../shared/services/df-paywall.service';
+import { DfSystemConfigDataService } from '../shared/services/df-system-config-data.service';
+import { DfDataMaskingApiService } from '../adf-data-masking/data-masking-api.service';
+import { MaskingOverview } from '../adf-data-masking/data-masking.types';
 
 // Secure & Manage pillar home (/api-security). Tiles mirror the section's
 // nav children: Authentication + Rate Limiting live here, Role-Based Access
@@ -30,7 +34,8 @@ import { ServiceType } from '../shared/types/service';
       [description]="t('description')"
       [actions]="actions"
       [groups]="groups"
-      [notes]="notes">
+      [notes]="notes"
+      [cardColumns]="3">
     
   `,
 })
@@ -38,11 +43,16 @@ export class DfSecurityOverviewComponent implements OnInit {
   private serviceTypeService = inject(SERVICE_TYPE_SERVICE_TOKEN);
   private transloco = inject(TranslocoService);
   private serviceTypes: ServiceType[] | null = null;
+  private paywall = inject(DfPaywallService);
+  private systemConfig = inject(DfSystemConfigDataService);
+  private maskingApi = inject(DfDataMaskingApiService);
+  /** Data Masking status line on its card; unset = no line (as before). */
+  private maskingMeta?: string;
 
   groups: SectionLandingGroup[] = [];
 
-  t(key: string): string {
-    return this.transloco.translate(`sectionOverviews.security.${key}`);
+  t(key: string, params?: Record): string {
+    return this.transloco.translate(`sectionOverviews.security.${key}`, params);
   }
 
   readonly actions: SectionLandingAction[] = [
@@ -90,6 +100,61 @@ export class DfSecurityOverviewComponent implements OnInit {
         this.serviceTypes = serviceTypes;
         this.buildGroups();
       });
+
+    this.loadMaskingStatus();
+  }
+
+  /**
+   * Data Masking's mode and queue on its card, where admins already look.
+   * Licenses without the feature make no request, and an instance without
+   * the package (paywalled, like Rate Limiting) shows no line either.
+   */
+  private loadMaskingStatus(): void {
+    const license =
+      this.systemConfig.environment?.platform?.license ?? 'OPEN SOURCE';
+    if (this.paywall.isFeatureLocked(ROUTES.DATA_MASKING, license)) return;
+    this.paywall
+      .activatePaywall('data_masking')
+      .pipe(
+        take(1),
+        switchMap(missing =>
+          missing
+            ? of(undefined)
+            : this.maskingApi.overview(true).pipe(
+                map(o => this.maskingStatus(o)),
+                catchError(() => of(undefined))
+              )
+        ),
+        catchError(() => of(undefined))
+      )
+      .subscribe(meta => {
+        this.maskingMeta = meta;
+        this.buildGroups();
+      });
+  }
+
+  private maskingStatus(o: MaskingOverview): string | undefined {
+    if (!o?.mode) return undefined;
+    if (o.scan?.status === 'running' && !o.scan.finishedAt) {
+      return this.t('maskingScanning');
+    }
+    if (!o.scan?.finishedAt && !o.scan?.columns) {
+      return this.t('maskingNotScanned');
+    }
+    const masked = o.counts?.masked ?? 0;
+    const review = o.counts?.review ?? 0;
+    switch (o.mode) {
+      case 'on':
+        return review
+          ? this.t('maskingOnReview', { masked, review })
+          : this.t('maskingOn', { masked });
+      case 'preview':
+        return review
+          ? this.t('maskingPreviewReview', { review })
+          : this.t('maskingPreview');
+      default:
+        return this.t('maskingOff');
+    }
   }
 
   private buildGroups(): void {
@@ -119,6 +184,23 @@ export class DfSecurityOverviewComponent implements OnInit {
             action: this.t('rateAction'),
             tint: 'system',
           },
+          {
+            icon: 'visibility_off',
+            title: this.t('maskingTitle'),
+            text: this.t('maskingText'),
+            route: `/${ROUTES.API_SECURITY}/${ROUTES.DATA_MASKING}`,
+            action: this.t('maskingAction'),
+            meta: this.maskingMeta,
+            tint: 'security',
+          },
+          {
+            icon: 'fact_check',
+            title: this.t('complianceTitle'),
+            text: this.t('complianceText'),
+            route: `/${ROUTES.API_SECURITY}/${ROUTES.COMPLIANCE}`,
+            action: this.t('complianceAction'),
+            tint: 'security',
+          },
           {
             icon: 'admin_panel_settings',
             title: this.t('rolesTitle'),
diff --git a/src/app/routes.ts b/src/app/routes.ts
index c2b4341ec..8cdc8521f 100644
--- a/src/app/routes.ts
+++ b/src/app/routes.ts
@@ -14,6 +14,8 @@ import {
   rolesResolver,
 } from './adf-roles/resolvers/role.resolver';
 import { limitsResolver } from './adf-limits/resolvers/limits.resolver';
+import { dataMaskingResolver } from './adf-data-masking/data-masking.resolver';
+import { complianceResolver } from './adf-compliance/compliance.resolver';
 import { serviceReportsResolver } from './adf-reports/resolvers/service-report.resolver';
 import { DfProfileService } from './adf-profile/services/df-profile.service';
 import { DfPasswordService } from './adf-user-management/services/df-password.service';
@@ -420,6 +422,24 @@ export const routes: Routes = [
         ],
         providers: [provideTranslocoScope('limits')],
       },
+      {
+        path: ROUTES.DATA_MASKING,
+        loadComponent: () =>
+          import('./adf-data-masking/df-data-masking/df-data-masking.component').then(
+            m => m.DfDataMaskingComponent
+          ),
+        resolve: { data: dataMaskingResolver },
+        providers: [provideTranslocoScope('dataMasking')],
+      },
+      {
+        path: ROUTES.COMPLIANCE,
+        loadComponent: () =>
+          import('./adf-compliance/df-compliance/df-compliance.component').then(
+            m => m.DfComplianceComponent
+          ),
+        resolve: { data: complianceResolver },
+        providers: [provideTranslocoScope('compliance')],
+      },
       {
         path: ROUTES.ROLE_BASED_ACCESS,
         redirectTo: `/${ROUTES.API_CONNECTIONS}/${ROUTES.ROLE_BASED_ACCESS}`,
diff --git a/src/app/shared/components/df-section-landing/df-section-landing.component.html b/src/app/shared/components/df-section-landing/df-section-landing.component.html
index a558c7160..b3be2a283 100644
--- a/src/app/shared/components/df-section-landing/df-section-landing.component.html
+++ b/src/app/shared/components/df-section-landing/df-section-landing.component.html
@@ -34,7 +34,10 @@ 
    {{ title }}
    
       *ngFor="let group of groups; trackBy: trackByGroupTitle"
       [attr.aria-label]="group.title">
       
    {{ group.title }}
    
-      
    
+      
    
         
           -* token pair via two local vars; the chip, action
 // link, and hover border below read those vars. No category = accent
diff --git a/src/app/shared/components/df-section-landing/df-section-landing.component.ts b/src/app/shared/components/df-section-landing/df-section-landing.component.ts
index cbaad3949..7e42b2350 100644
--- a/src/app/shared/components/df-section-landing/df-section-landing.component.ts
+++ b/src/app/shared/components/df-section-landing/df-section-landing.component.ts
@@ -76,6 +76,12 @@ export class DfSectionLandingComponent {
   @Input() stats: SectionLandingStat[] = [];
   @Input() groups: SectionLandingGroup[] = [];
   @Input() notes: SectionLandingNote[] = [];
+  /**
+   * Optional fixed card column count at desktop widths (e.g. 3 for a tidy
+   * 3+2 with five cards). Unset keeps the default auto-fit grid, so other
+   * overviews are unaffected. Narrow screens always fall back to auto-fit.
+   */
+  @Input() cardColumns: number | null = null;
 
   trackByGroupTitle(_: number, group: SectionLandingGroup): string {
     return group.title;
diff --git a/src/app/shared/services/action-registry.service.spec.ts b/src/app/shared/services/action-registry.service.spec.ts
new file mode 100644
index 000000000..ee8b93339
--- /dev/null
+++ b/src/app/shared/services/action-registry.service.spec.ts
@@ -0,0 +1,133 @@
+import { TestBed } from '@angular/core/testing';
+import { HttpClient } from '@angular/common/http';
+import { of } from 'rxjs';
+import { ActionRegistryService, PaletteItem } from './action-registry.service';
+
+/** Command palette: Data Masking and Compliance are found by what admins type. */
+describe('ActionRegistryService palette search', () => {
+  let registry: ActionRegistryService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({
+      providers: [{ provide: HttpClient, useValue: {} }],
+    });
+    registry = TestBed.inject(ActionRegistryService);
+  });
+
+  const ids = (q: string): string[] =>
+    registry
+      .query(q)
+      .flatMap(g => g.items)
+      .map((i: PaletteItem) => i.id);
+
+  it.each([
+    'pii',
+    'phi',
+    'pci',
+    'mask',
+    'masking',
+    'redact',
+    'personal data',
+    'hipaa',
+    'gdpr',
+  ])('"%s" finds Data Masking', q => {
+    expect(ids(q)).toContain('jump.data-masking');
+  });
+
+  it.each(['hipaa', 'pci', 'phi', 'gdpr', 'soc 2', 'soc2', 'iso', 'baseline'])(
+    '"%s" finds Compliance',
+    q => {
+      expect(ids(q)).toContain('jump.compliance');
+    }
+  );
+
+  it('ranks Data Masking first among pages for "pii"', () => {
+    const docs = registry.query('pii').find(g => g.kind === 'doc')!;
+    expect(docs.items[0].id).toBe('jump.data-masking');
+  });
+
+  it('keeps existing jumps where they were (rate limiting by "throttle")', () => {
+    const docs = registry.query('throttle').find(g => g.kind === 'doc')!;
+    expect(docs.items[0].id).toBe('jump.rate-limiting');
+  });
+
+  describe('ranking with real objects loaded (the GOLD stack)', () => {
+    // The objects dmgold lists: services, roles and apps whose names and
+    // keywords hold "p…i…i" / "p…c…i" as letters in order, but never the
+    // word itself.
+    const rows: Record = {
+      service: [
+        { id: 10, name: 'crm', label: 'CRM', type: 'pgsql' },
+        { id: 11, name: 'sports', label: 'Sports', type: 'pgsql' },
+        { id: 13, name: 'crm_mysql', label: 'CRM MySQL', type: 'mysql' },
+        { id: 15, name: 'hr_mssql', label: 'HR', type: 'sqlsrv' },
+        { id: 16, name: 'crm_mcp', label: 'CRM MCP', type: 'mcp' },
+        { id: 1, name: 'api_docs', label: 'Live API Docs', type: 'swagger' },
+      ],
+      role: [{ id: 1, name: 'support', description: 'Support desk' }],
+      app: [
+        { id: 4, name: 'mobile-app', description: 'Mobile application' },
+        { id: 1, name: 'admin', description: 'Admin application' },
+      ],
+    };
+
+    beforeEach(() => {
+      TestBed.resetTestingModule();
+      TestBed.configureTestingModule({
+        providers: [
+          {
+            provide: HttpClient,
+            useValue: {
+              get: (url: string) =>
+                of({ resource: rows[url.split('/').pop() ?? ''] ?? [] }),
+            },
+          },
+        ],
+      });
+      registry = TestBed.inject(ActionRegistryService);
+      registry.refresh().subscribe();
+    });
+
+    it.each(['pii', 'pci', 'phi', 'mask'])(
+      '"%s" puts Data Masking first, above fuzzy matches in other groups',
+      q => {
+        const groups = registry.query(q);
+        expect(groups[0].kind).toBe('doc');
+        expect(ids(q)[0]).toBe('jump.data-masking');
+      }
+    );
+
+    it('"hipaa" and "gdpr" open with the pages that name them', () => {
+      for (const q of ['hipaa', 'gdpr']) {
+        expect(registry.query(q)[0].kind).toBe('doc');
+        expect(ids(q).slice(0, 2).sort()).toEqual([
+          'jump.compliance',
+          'jump.data-masking',
+        ]);
+      }
+    });
+
+    it('fuzzy-only groups keep their order after the strong ones', () => {
+      const kinds = registry.query('pii').map(g => g.kind);
+      expect(kinds[0]).toBe('doc');
+      // The rest stay in the usual order: actions, services, …, keys.
+      const rest = kinds.slice(1);
+      const order = ['action', 'service', 'role', 'key', 'mcp'];
+      expect(rest).toEqual(order.filter(k => rest.includes(k as never)));
+    });
+
+    it('a label hit keeps the usual group order (actions first)', () => {
+      expect(registry.query('new').map(g => g.kind)[0]).toBe('action');
+      // "role": the New role verb (label) leads, as before.
+      expect(ids('role')[0]).toBe('verb.new-role');
+      expect(registry.query('role').map(g => g.kind)).toEqual([
+        'action',
+        'role',
+        'doc',
+      ]);
+      // A service whose label matches leads; verbs that only hold the
+      // letters in order follow.
+      expect(registry.query('crm')[0].kind).toBe('service');
+    });
+  });
+});
diff --git a/src/app/shared/services/action-registry.service.ts b/src/app/shared/services/action-registry.service.ts
index ccf8e0f94..82f46e68a 100644
--- a/src/app/shared/services/action-registry.service.ts
+++ b/src/app/shared/services/action-registry.service.ts
@@ -59,6 +59,8 @@ const RECENTS_MAX = 6;
 
 // Group render + fuzzy tie-break order. Actions first (a verb the operator
 // typed is almost always the intent), then objects, then plain nav jumps.
+// Groups whose only hits are fuzzy subsequence matches follow the groups
+// with a label or exact keyword hit, in the same order.
 const GROUP_ORDER: PaletteKind[] = [
   'action',
   'service',
@@ -68,6 +70,11 @@ const GROUP_ORDER: PaletteKind[] = [
   'doc',
 ];
 
+/** Match tiers, strongest first (see rank()). */
+const TIER_LABEL = 3;
+const TIER_KEYWORD = 2;
+const TIER_FUZZY = 1;
+
 const GROUP_LABEL: Record = {
   action: 'commandPalette.groups.actions',
   service: 'commandPalette.groups.services',
@@ -197,6 +204,24 @@ export class ActionRegistryService {
       path: `/${ROUTES.API_SECURITY}/${ROUTES.RATE_LIMITING}`,
       keywords: 'throttle quota 429 security',
     },
+    {
+      id: 'jump.data-masking',
+      kind: 'doc',
+      label: 'Data Masking',
+      path: `/${ROUTES.API_SECURITY}/${ROUTES.DATA_MASKING}`,
+      // Hidden match terms (never shown): what admins type for this,
+      // including the regulated-data acronyms the UI copy avoids.
+      keywords:
+        'pii phi pci mask masking redact sensitive personal data privacy anonymize obfuscate hide classification columns hipaa gdpr',
+    },
+    {
+      id: 'jump.compliance',
+      kind: 'doc',
+      label: 'Compliance',
+      path: `/${ROUTES.API_SECURITY}/${ROUTES.COMPLIANCE}`,
+      keywords:
+        'baseline hipaa phi pci dss gdpr soc 2 soc2 iso 27001 glba cjis audit evidence pii',
+    },
     {
       id: 'jump.ai',
       kind: 'doc',
@@ -327,24 +352,32 @@ export class ActionRegistryService {
       return [];
     }
     const scored = this.all()
-      .map(item => ({ item, score: this.score(q, item) }))
+      .map(item => ({ item, ...this.rank(q, item) }))
       .filter(entry => entry.score > 0)
-      .sort((a, b) => b.score - a.score);
+      .sort((a, b) => b.tier - a.tier || b.score - a.score);
 
     const byKind = new Map();
-    for (const { item } of scored) {
+    // Best tier per group: a group whose only hits are fuzzy subsequence
+    // matches goes after every group with a label or exact keyword hit.
+    const bestTier = new Map();
+    for (const { item, tier } of scored) {
       const bucket = byKind.get(item.kind) ?? [];
       // Cap each group so one huge object type cannot bury the others.
       if (bucket.length < 8) {
         bucket.push(item);
         byKind.set(item.kind, bucket);
       }
+      bestTier.set(item.kind, Math.max(bestTier.get(item.kind) ?? 0, tier));
     }
-    return GROUP_ORDER.filter(kind => byKind.has(kind)).map(kind => ({
-      kind,
-      labelKey: GROUP_LABEL[kind],
-      items: byKind.get(kind)!,
-    }));
+    const strong = (kind: PaletteKind) =>
+      (bestTier.get(kind) ?? 0) > TIER_FUZZY ? 0 : 1;
+    return GROUP_ORDER.filter(kind => byKind.has(kind))
+      .sort((a, b) => strong(a) - strong(b))
+      .map(kind => ({
+        kind,
+        labelKey: GROUP_LABEL[kind],
+        items: byKind.get(kind)!,
+      }));
   }
 
   /**
@@ -401,6 +434,26 @@ export class ActionRegistryService {
     return ids.filter((i): i is PaletteItem => !!i);
   }
 
+  /**
+   * How well an item matches: the tier first (label hit, then a whole-word
+   * keyword hit such as "pii", then a fuzzy subsequence), the score within
+   * it. An exact keyword is what an admin types for a page ("pii" for Data
+   * Masking), so it outranks letters that merely appear in order.
+   */
+  private rank(q: string, item: PaletteItem): { tier: number; score: number } {
+    const label = item.label.toLowerCase();
+    if (label.startsWith(q) || label.includes(q)) {
+      return { tier: TIER_LABEL, score: this.score(q, item) };
+    }
+    const keywords = ` ${(item.keywords ?? '').toLowerCase()} `;
+    const at = keywords.indexOf(` ${q} `);
+    if (at >= 0) {
+      // Earlier keywords are the item's main terms.
+      return { tier: TIER_KEYWORD, score: 1000 - at };
+    }
+    return { tier: TIER_FUZZY, score: this.score(q, item) };
+  }
+
   /**
    * Fuzzy subsequence score. Rewards a prefix hit, contiguous runs, and
    * word-boundary starts; a plain substring in keywords still scores. Returns
diff --git a/src/app/shared/services/df-paywall.service.spec.ts b/src/app/shared/services/df-paywall.service.spec.ts
new file mode 100644
index 000000000..79d9d29d9
--- /dev/null
+++ b/src/app/shared/services/df-paywall.service.spec.ts
@@ -0,0 +1,98 @@
+import { TestBed } from '@angular/core/testing';
+import { HttpClient } from '@angular/common/http';
+import {
+  ActivatedRouteSnapshot,
+  RouterStateSnapshot,
+  ResolveFn,
+} from '@angular/router';
+import { BehaviorSubject, Observable, firstValueFrom, of } from 'rxjs';
+import { DfPaywallService } from './df-paywall.service';
+import { DfSystemConfigDataService } from './df-system-config-data.service';
+import { DfErrorService } from './df-error.service';
+import { dataMaskingResolver } from 'src/app/adf-data-masking/data-masking.resolver';
+import { complianceResolver } from 'src/app/adf-compliance/compliance.resolver';
+
+/**
+ * Data Masking and Compliance ship in the df-masking package. Without its
+ * system resources they behave like every other commercial feature: the
+ * shared paywall, whatever the license.
+ */
+describe('DfPaywallService: Data Masking and Compliance', () => {
+  let config: {
+    system$: BehaviorSubject<{ resource: { name: string }[] }>;
+    environment$: BehaviorSubject>;
+    fetchSystemData: jest.Mock;
+    fetchEnvironmentData: jest.Mock;
+  };
+  let http: { post: jest.Mock };
+
+  function setup(license: string, resources: string[]) {
+    const system$ = new BehaviorSubject({
+      resource: resources.map(name => ({ name })),
+    });
+    config = {
+      system$,
+      environment$: new BehaviorSubject>({
+        platform: { license },
+      }),
+      fetchSystemData: jest.fn(() => of(system$.value)),
+      fetchEnvironmentData: jest.fn(),
+    };
+    http = { post: jest.fn(() => of({})) };
+    TestBed.configureTestingModule({
+      providers: [
+        DfPaywallService,
+        { provide: DfSystemConfigDataService, useValue: config },
+        { provide: DfErrorService, useValue: {} },
+        { provide: HttpClient, useValue: http },
+      ],
+    });
+    return TestBed.inject(DfPaywallService);
+  }
+
+  const resolve = (fn: ResolveFn<'paywall' | 'ok'>) =>
+    firstValueFrom(
+      TestBed.runInInjectionContext(
+        () =>
+          fn(
+            {} as ActivatedRouteSnapshot,
+            {} as RouterStateSnapshot
+          ) as Observable<'paywall' | 'ok'>
+      )
+    );
+
+  afterEach(() => TestBed.resetTestingModule());
+
+  it.each(['OPEN SOURCE', 'SILVER', 'GOLD'])(
+    'shows the paywall on %s when the package is absent',
+    async license => {
+      setup(license, ['user', 'limit']);
+      expect(await resolve(dataMaskingResolver)).toBe('paywall');
+      expect(await resolve(complianceResolver)).toBe('paywall');
+      // The decision itself sends nothing; the paywall component does.
+      expect(http.post).not.toHaveBeenCalled();
+      expect(config.fetchEnvironmentData).not.toHaveBeenCalled();
+    }
+  );
+
+  it('opens each page when its own system resource exists', async () => {
+    setup('GOLD', ['data_masking', 'security_baseline', 'user']);
+    expect(await resolve(dataMaskingResolver)).toBe('ok');
+    expect(await resolve(complianceResolver)).toBe('ok');
+  });
+
+  it('checks each resource separately', async () => {
+    setup('GOLD', ['data_masking']);
+    expect(await resolve(dataMaskingResolver)).toBe('ok');
+    expect(await resolve(complianceResolver)).toBe('paywall');
+  });
+
+  it('locks both features in the nav on open source and silver, not on GOLD', () => {
+    const svc = setup('GOLD', []);
+    for (const route of ['data-masking', 'compliance']) {
+      expect(svc.isFeatureLocked(route, 'OPEN SOURCE')).toBe(true);
+      expect(svc.isFeatureLocked(route, 'SILVER')).toBe(true);
+      expect(svc.isFeatureLocked(route, 'GOLD')).toBe(false);
+    }
+  });
+});
diff --git a/src/app/shared/services/df-paywall.service.ts b/src/app/shared/services/df-paywall.service.ts
index ea6fbddbd..8efacfa0c 100644
--- a/src/app/shared/services/df-paywall.service.ts
+++ b/src/app/shared/services/df-paywall.service.ts
@@ -14,9 +14,17 @@ export class DfPaywallService {
     'rate-limiting',
     'scheduler',
     'reporting',
+    'data-masking',
+    'compliance',
   ];
 
-  private silverLockedFeatures = ['rate-limiting', 'scheduler', 'reporting'];
+  private silverLockedFeatures = [
+    'rate-limiting',
+    'scheduler',
+    'reporting',
+    'data-masking',
+    'compliance',
+  ];
 
   isFeatureLocked(route: string, licenseType: string): boolean {
     if (licenseType == 'GOLD') return false;
diff --git a/src/app/shared/types/routes.ts b/src/app/shared/types/routes.ts
index 3328b3a03..f38c58fe5 100644
--- a/src/app/shared/types/routes.ts
+++ b/src/app/shared/types/routes.ts
@@ -41,6 +41,8 @@ export enum ROUTES {
   DATA_EXPLORER = 'data-explorer',
   API_SECURITY = 'api-security',
   RATE_LIMITING = 'rate-limiting',
+  DATA_MASKING = 'data-masking',
+  COMPLIANCE = 'compliance',
   AUTHENTICATION = 'authentication',
   SYSTEM_SETTINGS = 'system-settings',
   CONFIG = 'config',
diff --git a/src/assets/i18n/compliance/en.json b/src/assets/i18n/compliance/en.json
new file mode 100644
index 000000000..55ec6490f
--- /dev/null
+++ b/src/assets/i18n/compliance/en.json
@@ -0,0 +1,55 @@
+{
+  "retry": "Retry",
+  "cancel": "Cancel",
+  "baselines": "Baselines",
+  "baseline": "Baseline",
+  "title": "{{name}} baseline",
+  "mapped": "Mapped to {{frameworks}}",
+  "notCertification": "Not a certification",
+  "scanning": "Scanning databases…",
+  "tracking": "Tracking",
+  "trackingSince": "Tracking since {{when}}",
+  "of": "of {{total}}",
+  "scoreLabel": "{{pass}} of {{total}} checks pass",
+  "checks": "Checks",
+  "apply": "Apply",
+  "changesN": "{{count}} changes",
+  "export": "Export",
+  "exportJson": "JSON",
+  "exportCsv": "CSV",
+  "stopTracking": "Stop tracking",
+  "fix": "Fix",
+  "fixFor": "Fix: {{title}}",
+  "manual": "Manual checks",
+  "status": {
+    "pass": "Passes",
+    "fail": "Fails",
+    "manual": "Check manually",
+    "na": "Not applicable"
+  },
+  "verticals": {
+    "healthcare": "Healthcare",
+    "payments": "Payments",
+    "banking": "Banking",
+    "government": "Government",
+    "privacy": "Privacy",
+    "general": "General"
+  },
+  "applyDialog": {
+    "title": "Apply {{name}} baseline?",
+    "none": "Your settings already match. Applying starts tracking this baseline.",
+    "turnOn": "Mask values now (masking On)",
+    "confirm": "Apply"
+  },
+  "toast": {
+    "applied": "{{name}} baseline applied",
+    "appliedOn": "{{name}} baseline applied. Masking is on.",
+    "appliedScanning": "{{name}} baseline applied. Scanning…",
+    "stopped": "Stopped tracking {{name}}"
+  },
+  "empty": {
+    "loadFailed": "Couldn't load compliance baselines",
+    "detailFailed": "Couldn't load this baseline",
+    "none": "No baselines"
+  }
+}
diff --git a/src/assets/i18n/dataExplorer/en.json b/src/assets/i18n/dataExplorer/en.json
index 919d727f6..4a2cf9a1f 100644
--- a/src/assets/i18n/dataExplorer/en.json
+++ b/src/assets/i18n/dataExplorer/en.json
@@ -32,5 +32,11 @@
   "showApiCall": "Show API Call",
   "quickSearch": "Search rows...",
   "searchDatabases": "Search databases",
-  "noMatches": "No databases match"
+  "noMatches": "No databases match",
+  "masked": "Masked",
+  "maskedTip": "Data Masking hides values in some columns",
+  "maskingFilter": "Masked columns can't be filtered: {{cols}}",
+  "maskingFilterAll": "Masked columns can't be filtered",
+  "maskingSort": "Sort ignored on masked columns: {{cols}}",
+  "maskingMore": "+{{count}} more"
 }
diff --git a/src/assets/i18n/dataMasking/en.json b/src/assets/i18n/dataMasking/en.json
new file mode 100644
index 000000000..700728e6d
--- /dev/null
+++ b/src/assets/i18n/dataMasking/en.json
@@ -0,0 +1,279 @@
+{
+  "notSensitive": "Not sensitive",
+  "cancel": "Cancel",
+  "retry": "Retry",
+  "strip": "{{masked}} masked, {{review}} to review, {{clear}} not sensitive",
+  "mode": {
+    "label": "Masking mode",
+    "off": "Off",
+    "preview": "Preview",
+    "on": "On",
+    "confirmOn": {
+      "title": "Turn on masking?",
+      "message": "{{columns}} in {{databases}} will be masked for apps, AI tools and admins, you included.",
+      "messageWithAccess": "{{columns}} in {{databases}} will be masked for apps, AI tools and admins without access.",
+      "review": "{{review}} still to review.",
+      "confirm": "Turn on"
+    },
+    "confirmOff": {
+      "title": "Turn off masking?",
+      "message": "Apps, AI tools and admins will receive real values.",
+      "confirm": "Turn off"
+    },
+    "locked": "Only the super admin can lower the masking mode.",
+    "confirmPreview": {
+      "title": "Stop masking?",
+      "message": "Apps, AI tools and admins will receive real values. Preview only reports what would be masked.",
+      "confirm": "Switch to Preview"
+    }
+  },
+  "counts": {
+    "masked": "masked",
+    "review": "to review"
+  },
+  "tabs": {
+    "columns": "Columns",
+    "access": "Access",
+    "masks": "Masks",
+    "detection": "Detection",
+    "activity": "Activity",
+    "label": "Section"
+  },
+  "filters": {
+    "review": "To review",
+    "masked": "Masked",
+    "clear": "Not sensitive",
+    "all": "All"
+  },
+  "columns": {
+    "search": "Search tables and columns",
+    "clearSearch": "Clear search",
+    "clearFilter": "Clear filter",
+    "filter": "Filter columns",
+    "scan": "Scan now",
+    "scanning": "Scanning {{progress}}%",
+    "table": "Table",
+    "tables": "Tables",
+    "column": "Column",
+    "class": "Class",
+    "maskedAs": "Masked as",
+    "decision": "Decision",
+    "evidence": "Evidence",
+    "evidenceFor": "Evidence for {{column}}",
+    "keep": "Keep masked",
+    "next": "Next table",
+    "reviewed": "Reviewed",
+    "allReviewed": "All reviewed",
+    "scanningNoProgress": "Scanning…",
+    "scanProgress": "Scan progress",
+    "notScanned": "Not scanned yet",
+    "scanUnknown": "Couldn't read the scan history. Check the DreamFactory log.",
+    "scanShort": "Scan",
+    "scanService": "Scan {{name}}",
+    "uncovered": "Not covered",
+    "uncoveredTip": "Data Masking can't mask this database. AI tools can't read it while masking is on.",
+    "scanFailed": "Couldn't read this database",
+    "failed": "Couldn't scan",
+    "retryService": "Retry {{name}}",
+    "nothingSensitive": "Nothing sensitive",
+    "noTables": "No tables"
+  },
+  "verdict": {
+    "masked": "Masked",
+    "review": "Needs review",
+    "clear": "Not sensitive"
+  },
+  "ledger": {
+    "score": "Score",
+    "type": "Type",
+    "shapes": "Value shapes",
+    "decision": "Decision",
+    "setBy": "Set by {{name}} {{when}}",
+    "setByYou": "Set by you {{when}}"
+  },
+  "empty": {
+    "noScan": "No scan yet",
+    "scanAction": "Scan databases",
+    "noDatabases": "No SQL databases yet",
+    "addDatabase": "Add a database",
+    "reviewed": "All reviewed",
+    "search": "No columns match \"{{q}}\"",
+    "loadFailed": "Couldn't load data masking",
+    "filter": {
+      "masked": "No masked columns",
+      "clear": "No columns marked not sensitive",
+      "all": "No columns",
+      "review": "All reviewed"
+    },
+    "sliceFailed": "Couldn't load this tab",
+    "noTables": "No tables to scan"
+  },
+  "firstRun": {
+    "baselines": "Start from a baseline"
+  },
+  "access": {
+    "title": "Who sees real values",
+    "locked": "Only the super admin can give access.",
+    "admins": "Admins",
+    "none": "No one",
+    "selected": "Selected",
+    "allAdmins": "All admins",
+    "you": "(you)",
+    "expiryFor": "Access for {{name}} lasts",
+    "until": "until {{when}}",
+    "durations": {
+      "1h": "1 hour",
+      "8h": "8 hours",
+      "24h": "24 hours",
+      "7d": "7 days",
+      "forever": "Until turned off"
+    },
+    "ai": "AI and MCP sessions",
+    "aiMasked": "Always masked",
+    "aiSame": "Same as admins",
+    "roles": "Roles",
+    "noRoles": "No roles",
+    "scopeFor": "Real values {{name}} sees",
+    "scopeAll": "All classes",
+    "scopeExceptRestricted": "All except restricted",
+    "exceeds": "Exceeds {{name}}",
+    "confirmAll": {
+      "title": "Give every admin real values?",
+      "message": "All admins will see real values in {{columns}}.",
+      "confirm": "Give access"
+    },
+    "confirmAi": {
+      "title": "Let AI sessions see real values?",
+      "message": "AI and MCP sessions will see real values wherever their admin does.",
+      "confirm": "Allow"
+    }
+  },
+  "masks": {
+    "formatFor": "Mask format for {{cls}}"
+  },
+  "detection": {
+    "title": "Custom rules",
+    "addRule": "Add rule",
+    "match": "Match",
+    "matchName": "Column name",
+    "matchValue": "Value pattern",
+    "pattern": "Pattern",
+    "patternNameExample": "^emp_ssn$|social_sec",
+    "patternValueExample": "^[A-Z]{2}\\d{6}$",
+    "pickClass": "Choose a class",
+    "matches": "Matches",
+    "matchesN": "Matches {{columns}}",
+    "more": "and {{count}} more",
+    "showMatches": "Show {{count}} matching columns",
+    "invalid": "Not a valid pattern",
+    "delete": "Delete rule",
+    "empty": "No custom rules",
+    "builtIn": "Built-in rules",
+    "matchesAtLeast": "Matches at least {{columns}}",
+    "fromBaseline": "Added by the {{name}} baseline",
+    "deleteTitle": "Delete rule?",
+    "deleteKept": "{{columns}} it masked stay masked.",
+    "deleteConfirm": "Delete rule",
+    "deleteUnmask": "Delete and unmask {{count}}"
+  },
+  "activity": {
+    "filter": "Filter activity",
+    "today": "Today",
+    "yesterday": "Yesterday",
+    "empty": "No activity yet",
+    "filters": {
+      "all": "All",
+      "changes": "Changes",
+      "access": "Access",
+      "scans": "Scans",
+      "blocked": "Blocked",
+      "preview": "Preview"
+    },
+    "older": "Load older"
+  },
+  "toast": {
+    "mode": {
+      "on": "Masking turned on",
+      "off": "Masking turned off",
+      "preview": "Preview turned on",
+      "paused": "Masking stopped. Preview is on."
+    },
+    "access": "Access updated",
+    "accessStale": "Access settings changed elsewhere. Showing the latest.",
+    "mask": "Mask updated",
+    "ruleAdded": "Rule added",
+    "ruleDeleted": "Rule deleted",
+    "scanDone": "Scan finished",
+    "scanDonePreview": "Scan finished. Preview is on.",
+    "scanPartial": "Scan finished. Couldn't read {{label}}.",
+    "scanFailed": "Scan failed. Couldn't read {{label}}.",
+    "scanFailedNoReason": "Scan didn't finish.",
+    "notSensitive": "{{column}} marked not sensitive",
+    "ruleDeletedKept": "Rule deleted. {{columns}} stay masked.",
+    "ruleDeletedUnmasked": "Rule deleted. {{columns}} unmasked.",
+    "ruleUnmaskPartial": "Rule deleted. Unmasked {{done}} of {{total}} columns."
+  },
+  "mv": {
+    "masked": "Masked",
+    "endsIn": "Masked, ends in {{tail}}",
+    "startsWith": "Masked, starts with {{head}}",
+    "token": "Token {{token}}",
+    "null": "Null",
+    "nullValue": "Masked, returned as null"
+  },
+  "tiers": {
+    "restricted": "Restricted",
+    "identity": "Identity",
+    "quasi": "Quasi-identifiers",
+    "free_text": "Free text"
+  },
+  "formats": {
+    "redact": "Redact",
+    "last4": "Last 4",
+    "initial": "First letter",
+    "year": "Year only",
+    "token": "Token",
+    "null": "Null",
+    "span": "Detected spans"
+  },
+  "classes": {
+    "national_id": "National ID",
+    "tax_id": "Tax ID",
+    "passport_number": "Passport number",
+    "drivers_license": "Driver's license",
+    "card_number": "Card number",
+    "card_security": "Card security code",
+    "bank_account": "Bank account",
+    "credential": "Credential",
+    "special_category": "Health & special category",
+    "email": "Email",
+    "phone": "Phone",
+    "person_name": "Person name",
+    "street_address": "Street address",
+    "date_of_birth": "Date of birth",
+    "ip_address": "IP address",
+    "username": "Username",
+    "geo_precise": "Precise location",
+    "postal_code": "Postal code",
+    "gender": "Gender",
+    "age": "Age",
+    "card_expiry": "Card expiry",
+    "financial_personal": "Personal financial",
+    "free_text_pii": "Free text with personal data"
+  },
+  "plural": {
+    "columns": {
+      "one": "{{count}} column",
+      "other": "{{count}} columns"
+    },
+    "databases": {
+      "one": "{{count}} database",
+      "other": "{{count}} databases"
+    }
+  },
+  "undo": "Undo",
+  "list": {
+    "two": "{{a}} and {{b}}",
+    "more": "{{a}} and {{count}} more"
+  }
+}
diff --git a/src/assets/i18n/en.json b/src/assets/i18n/en.json
index 643be85f6..9cc5df936 100644
--- a/src/assets/i18n/en.json
+++ b/src/assets/i18n/en.json
@@ -388,6 +388,19 @@
       "rateTitle": "Rate Limiting",
       "rateText": "Cap how many calls a user, role, service, IP, or key can make so no single client can overwhelm your APIs.",
       "rateAction": "Manage rate limits",
+      "maskingTitle": "Data Masking",
+      "maskingText": "Find sensitive columns and mask their values for apps, roles, and AI tools.",
+      "maskingAction": "Manage data masking",
+      "maskingNotScanned": "Not scanned",
+      "maskingScanning": "Scanning",
+      "maskingOff": "Off",
+      "maskingPreview": "Preview",
+      "maskingPreviewReview": "Preview · {{review}} to review",
+      "maskingOn": "On · {{masked}} masked",
+      "maskingOnReview": "On · {{masked}} masked · {{review}} to review",
+      "complianceTitle": "Compliance",
+      "complianceText": "Apply a baseline for HIPAA, PCI DSS, GDPR, SOC 2 and more, then track every check.",
+      "complianceAction": "Manage compliance",
       "rolesTitle": "Role-Based Access",
       "rolesText": "Define exactly which services, tables, and endpoints each caller can reach.",
       "rolesAction": "Manage roles",
@@ -604,6 +617,14 @@
           "header": "Edit Limit"
         }
       },
+      "data-masking": {
+        "nav": "Data Masking",
+        "header": "{{nav.api-security.data-masking.nav}}"
+      },
+      "compliance": {
+        "nav": "Compliance",
+        "header": "{{nav.api-security.compliance.nav}}"
+      },
       "authentication": {
         "nav": "Authentication",
         "header": "{{nav.api-security.authentication.nav}}",
diff --git a/src/styles.scss b/src/styles.scss
index 9a2c0899a..8b655ded3 100644
--- a/src/styles.scss
+++ b/src/styles.scss
@@ -97,6 +97,18 @@ $df-purple-palette: mat.define-palette(theme.$df-purple-palette);
   --df-tint-docs-bg: rgba(13, 148, 136, 0.1);
   --df-tint-docs-fg: #0d9488;
 
+  /* Data Masking tiers: redaction bars, class chips, sensitivity strips.
+     Kept clear of the danger red and the brand purple; each has a wash. */
+  --df-mask-restricted: #be123c;
+  --df-mask-restricted-soft: rgba(190, 18, 60, 0.09);
+  --df-mask-identity: #a16207;
+  --df-mask-identity-soft: rgba(161, 98, 7, 0.1);
+  --df-mask-quasi: #4f6d8f;
+  --df-mask-quasi-soft: rgba(79, 109, 143, 0.11);
+  --df-mask-free-text: #0f766e;
+  --df-mask-free-text-soft: rgba(15, 118, 110, 0.1);
+  --df-mask-clear: var(--df-border);
+
   /* ============================================================
      MERIDIAN PHASE 0 - net-new token layers (spec 1.1-1.6).
      Type/space/motion/focus are theme-agnostic and live only here;
@@ -222,6 +234,17 @@ $df-purple-palette: mat.define-palette(theme.$df-purple-palette);
   --df-tint-docs-bg: rgba(45, 212, 191, 0.12);
   --df-tint-docs-fg: #2dd4bf;
 
+  /* Data Masking tiers: same hues, lifted for the dark canvas. */
+  --df-mask-restricted: #fb7185;
+  --df-mask-restricted-soft: rgba(251, 113, 133, 0.14);
+  --df-mask-identity: #e3b35c;
+  --df-mask-identity-soft: rgba(227, 179, 92, 0.14);
+  --df-mask-quasi: #97b0cc;
+  --df-mask-quasi-soft: rgba(151, 176, 204, 0.15);
+  --df-mask-free-text: #5eead4;
+  --df-mask-free-text-soft: rgba(94, 234, 212, 0.12);
+  --df-mask-clear: var(--df-border);
+
   /* Meridian per-theme (spec 1.3-1.5): only overlay/warning/skeleton
      shift for dark; type/space/motion/focus inherit from :root. */
   --df-shadow-overlay:
@@ -294,6 +317,26 @@ $df-purple-palette: mat.define-palette(theme.$df-purple-palette);
   --df-tint-docs-bg: rgba(51, 255, 102, 0.08);
   --df-tint-docs-fg: #2edd5b;
 
+  /* Data Masking tiers in phosphor. Amber stays warnings-only, so the
+     tiers part by luminance and texture instead of hue: restricted is the
+     full-bright solid, identity a hatched mid green, quasi a dim solid,
+     free text the Blade Runner cyan. Restricted also carries a lock. */
+  --df-mask-restricted: #39ff6a;
+  --df-mask-restricted-soft: rgba(57, 255, 106, 0.16);
+  --df-mask-identity: #2edd5b;
+  --df-mask-identity-soft: rgba(46, 221, 91, 0.1);
+  --df-mask-identity-fill: repeating-linear-gradient(
+    135deg,
+    #2edd5b 0 2px,
+    rgba(46, 221, 91, 0.25) 2px 4px
+  );
+  --df-mask-quasi: #23a847;
+  --df-mask-quasi-soft: rgba(35, 168, 71, 0.16);
+  --df-mask-quasi-fill: rgba(35, 168, 71, 0.5);
+  --df-mask-free-text: #4dd8e6;
+  --df-mask-free-text-soft: rgba(77, 216, 230, 0.1);
+  --df-mask-clear: var(--df-border);
+
   /* Meridian per-theme (spec 1.3-1.4): no drop shadow on a terminal -
      the overlay is a border-glow. Amber is warnings-only in phosphor
      (Captain rule), so the warning trio rides the phosphor amber. */