diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 9e59901692f..69bc73cf5dd 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -56,6 +56,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up JDK 11 @@ -96,6 +98,10 @@ jobs: if: matrix.mode == 'anonymous' working-directory: zeppelin-web-angular run: ./node/npm run check:websocket-contract + - name: Check notebook parity scenarios + if: matrix.mode == 'anonymous' + working-directory: zeppelin-web-angular + run: ./node/npm run check:notebook-parity-scenarios # Keeps the Karma coverage previously run by the removed zeppelin-web e2e job - name: Run zeppelin-web unit tests if: matrix.mode == 'anonymous' diff --git a/zeppelin-web-angular/e2e/AGENTS.md b/zeppelin-web-angular/e2e/AGENTS.md index 9b76c1f3874..35a7da5d978 100644 --- a/zeppelin-web-angular/e2e/AGENTS.md +++ b/zeppelin-web-angular/e2e/AGENTS.md @@ -117,11 +117,12 @@ Use an existing key from the `PAGES` object in `e2e/utils.ts`; add a new one the 1. Pick/confirm the target route and the `PAGES` key. 2. Copy the shape of an existing spec in the same ``; reuse or extend the matching POM (`models/.ts` + `.util.ts`). Do not inline selectors the POM already owns. 3. Annotate the page (`addPageAnnotationBeforeEach`), navigate, then `waitForZeppelinReady`. -4. Run `npm run e2e:fast` and iterate until green. +4. If the test covers a scenario in `e2e/scenarios/notebook-parity.json`, add its stable ID as a Playwright tag such as `{ tag: '@NB-PARITY-001' }`. Keep the title human-readable; the registry links coverage by tag and path. Browser execution controls such as project lists and skip conditions stay in the spec rather than being copied into the registry. Keep browser assumptions in the registry only when they define the scenario's behavior or expected outcome. +5. Run `npm run e2e:fast` and iterate until green. ## Migration (Angular to React Microfrontend) -Pages are moving from Angular to React fragments incrementally. Today this is narrow: the published paragraph route reads a `?react=true` flag (`published/paragraph/paragraph.component`), the notebook footer swaps via a `?reactFooter=true` flag (read into the notebook component's `useReactFooter` input), and the configuration table swaps via a `?reactConfiguration=true` flag (`configuration/configuration.component`). All three are query params inside the hash. There is no app-wide "flip this route to React" flag, and no cross-framework parity project in this config. Write specs so they survive a route being reimplemented, but do not build parity infrastructure ahead of need. +Pages are moving from Angular to React fragments incrementally. Today this is narrow: the published paragraph route reads a `?react=true` flag (`published/paragraph/paragraph.component`), the notebook footer swaps via a `?reactFooter=true` flag (read into the notebook component's `useReactFooter` input), and the configuration table swaps via a `?reactConfiguration=true` flag (`configuration/configuration.component`). All three are query params inside the hash. There is no app-wide "flip this route to React" flag and no separate cross-framework Playwright project in this config. The notebook parity registry records the Angular behavior baseline and links it to existing framework-neutral tests; add scenarios as migration work reaches them rather than duplicating the suite for both frameworks. ### Write Framework-Neutral Specs diff --git a/zeppelin-web-angular/e2e/scenarios/notebook-parity.json b/zeppelin-web-angular/e2e/scenarios/notebook-parity.json new file mode 100644 index 00000000000..9ee808f9078 --- /dev/null +++ b/zeppelin-web-angular/e2e/scenarios/notebook-parity.json @@ -0,0 +1,546 @@ +{ + "$schema": "./notebook-parity.schema.json", + "reviewedCommit": "d5b57b12fd0c5e1d885767aabe06b242debf8300", + "scenarios": [ + { + "id": "NB-PARITY-001", + "name": "Notebook container structure is visible", + "area": "navigation", + "preconditions": ["A disposable notebook route is open."], + "action": "Render the notebook route.", + "observableOutcomes": [ + { + "id": "NB-PARITY-001-OUTCOME-001", + "description": "The notebook container is visible with the expected container class." + } + ], + "interpreter": null, + "coverage": { + "status": "covered", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts", + "tag": "@NB-PARITY-001" + } + ], + "issues": [], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts", + "symbol": "Notebook Container Component" + }, + { + "path": "zeppelin-web-angular/e2e/models/notebook-page.ts", + "symbol": "NotebookPage" + } + ] + }, + { + "id": "NB-PARITY-002", + "name": "Notebook title can be displayed and edited", + "area": "navigation", + "preconditions": ["A disposable notebook route is open."], + "action": "Open the title editor and rename the notebook.", + "observableOutcomes": [ + { + "id": "NB-PARITY-002-OUTCOME-001", + "description": "The title editor is visible and the changed title is reflected in the notebook header." + } + ], + "interpreter": null, + "roleExpectations": { + "owner": "allow", + "writer": "allow", + "reader": "deny", + "runner": "not-applicable" + }, + "roleVerification": { + "owner": "unverified", + "writer": "unverified", + "reader": "unverified", + "runner": "not-applicable" + }, + "coverage": { + "status": "covered", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts", + "tag": "@NB-PARITY-002" + } + ], + "issues": [], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts", + "symbol": "Notebook Action Bar Functionality" + }, + { + "path": "zeppelin-web-angular/e2e/models/notebook-action-bar-page.ts", + "symbol": "NotebookActionBarPage" + } + ] + }, + { + "id": "NB-PARITY-003", + "name": "Paragraph enters editing mode on double click", + "area": "editor", + "preconditions": ["A disposable notebook with at least one paragraph is open."], + "action": "Double-click the paragraph.", + "observableOutcomes": [ + { + "id": "NB-PARITY-003-OUTCOME-001", + "description": "The code editor becomes visible for the paragraph." + } + ], + "interpreter": null, + "roleExpectations": { + "owner": "allow", + "writer": "allow", + "reader": "deny", + "runner": "not-applicable" + }, + "roleVerification": { + "owner": "unverified", + "writer": "unverified", + "reader": "unverified", + "runner": "not-applicable" + }, + "coverage": { + "status": "covered", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts", + "tag": "@NB-PARITY-003" + } + ], + "issues": [], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts", + "symbol": "Notebook Paragraph Functionality" + }, + { + "path": "zeppelin-web-angular/e2e/models/notebook-paragraph-page.ts", + "symbol": "NotebookParagraphPage" + } + ] + }, + { + "id": "NB-PARITY-004", + "name": "Paragraph add buttons are visible", + "area": "editor", + "preconditions": ["A disposable notebook with a paragraph is open."], + "action": "Inspect the paragraph controls.", + "observableOutcomes": [ + { + "id": "NB-PARITY-004-OUTCOME-001", + "description": "Add-paragraph controls are visible where a user can add another paragraph." + } + ], + "interpreter": null, + "roleExpectations": { + "owner": "allow", + "writer": "allow", + "reader": "deny", + "runner": "not-applicable" + }, + "roleVerification": { + "owner": "unverified", + "writer": "unverified", + "reader": "unverified", + "runner": "not-applicable" + }, + "coverage": { + "status": "covered", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts", + "tag": "@NB-PARITY-004" + } + ], + "issues": [], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts", + "symbol": "Notebook Paragraph Functionality" + }, + { + "path": "zeppelin-web-angular/e2e/models/notebook-paragraph-page.ts", + "symbol": "NotebookParagraphPage" + } + ] + }, + { + "id": "NB-PARITY-005", + "name": "Shift+Enter executes a markdown paragraph", + "area": "shortcut", + "preconditions": ["A disposable notebook paragraph is focused in the code editor."], + "action": "Type Markdown content and press Shift+Enter.", + "observableOutcomes": [ + { + "id": "NB-PARITY-005-OUTCOME-001", + "description": "The paragraph executes and renders the Markdown heading result." + } + ], + "interpreter": "md", + "roleExpectations": { + "owner": "allow", + "writer": "allow", + "reader": "deny", + "runner": "allow" + }, + "roleVerification": { + "owner": "unverified", + "writer": "unverified", + "reader": "unverified", + "runner": "unverified" + }, + "coverage": { + "status": "covered", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts", + "tag": "@NB-PARITY-005" + } + ], + "issues": [], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts", + "symbol": "ParagraphActions.Run" + }, + { + "path": "zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts", + "symbol": "NotebookKeyboardPage" + } + ] + }, + { + "id": "NB-PARITY-010", + "name": "History inline completion can be dismissed without losing editor focus", + "area": "editor", + "preconditions": [ + "A notebook contains a Python paragraph with prior text that can seed inline completion.", + "The notebook route is opened with aiInlineComplete enabled." + ], + "action": "Type a completion prefix in Monaco and press Escape while the completion is visible.", + "observableOutcomes": [ + { + "id": "NB-PARITY-010-OUTCOME-001", + "description": "The inline completion suggestion is shown from notebook history." + }, + { + "id": "NB-PARITY-010-OUTCOME-002", + "description": "The Monaco input remains focused after the first Escape dismisses the suggestion." + } + ], + "interpreter": "python", + "coverage": { + "status": "covered", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts", + "tag": "@NB-PARITY-010" + } + ], + "issues": [], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts", + "symbol": "Inline completion" + }, + { + "path": "zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts", + "symbol": "NotebookKeyboardPage" + } + ] + }, + { + "id": "NB-PARITY-011", + "name": "The second Escape after inline completion dismissal blurs the editor", + "area": "editor", + "preconditions": [ + "A notebook contains a Python paragraph with prior text that can seed inline completion.", + "The notebook route is opened with aiInlineComplete enabled.", + "The browser is Chromium." + ], + "action": "Press Escape once to dismiss the completion and press Escape again.", + "observableOutcomes": [ + { + "id": "NB-PARITY-011-OUTCOME-001", + "description": "The first Escape keeps Monaco focused." + }, + { + "id": "NB-PARITY-011-OUTCOME-002", + "description": "The second Escape blurs the Monaco input in Chromium." + } + ], + "interpreter": "python", + "coverage": { + "status": "covered", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts", + "tag": "@NB-PARITY-011" + } + ], + "issues": [], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts", + "symbol": "Inline completion" + }, + { + "path": "zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts", + "symbol": "NotebookKeyboardPage" + } + ] + }, + { + "id": "NB-PARITY-021", + "name": "Text and table result displays preserve output semantics after paragraph execution", + "area": "result", + "preconditions": [ + "A notebook has a Python paragraph that prints text.", + "A notebook has a paragraph that returns tabular output." + ], + "action": "Run the paragraph from the paragraph control and inspect the rendered result panel.", + "observableOutcomes": [ + { + "id": "NB-PARITY-021-OUTCOME-001", + "description": "The result display becomes visible and is not empty." + }, + { + "id": "NB-PARITY-021-OUTCOME-002", + "description": "The UI offers every display mode that the Angular notebook exposes for the returned result type." + }, + { + "id": "NB-PARITY-021-OUTCOME-003", + "description": "Visualization control changes preserve the field mapping from result columns to configured dimensions or measures." + }, + { + "id": "NB-PARITY-021-OUTCOME-004", + "description": "The paragraph's persisted config reflects the resulting configuration object after a visualization option changes." + }, + { + "id": "NB-PARITY-021-OUTCOME-005", + "description": "Text and table results expose accessible table output row by row so migrated React rendering can be compared without relying on screenshots alone." + } + ], + "interpreter": "python", + "roleExpectations": { + "owner": "allow", + "writer": "allow", + "reader": "deny", + "runner": "allow" + }, + "roleVerification": { + "owner": "unverified", + "writer": "unverified", + "reader": "unverified", + "runner": "unverified" + }, + "coverage": { + "status": "partial", + "tests": [ + { + "path": "zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts", + "tag": "@NB-PARITY-021" + } + ], + "issues": ["ZEPPELIN-6514", "ZEPPELIN-6516"], + "uncoveredOutcomes": [ + "NB-PARITY-021-OUTCOME-002", + "NB-PARITY-021-OUTCOME-003", + "NB-PARITY-021-OUTCOME-004", + "NB-PARITY-021-OUTCOME-005" + ] + }, + "implementationEvidence": [ + { + "path": "zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/paragraph.component.html", + "symbol": "zeppelin-notebook-paragraph-result" + }, + { + "path": "zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/progress/progress.component.ts", + "symbol": "NotebookParagraphProgressComponent" + }, + { + "path": "zeppelin-web-angular/projects/zeppelin-visualization/src/table-transformation.ts", + "symbol": "TableTransformation" + }, + { + "path": "zeppelin-web-angular/projects/zeppelin-visualization/src/pivot-transformation.ts", + "symbol": "PivotTransformation" + }, + { + "path": "zeppelin-web-angular/projects/zeppelin-visualization/src/visualization.ts", + "symbol": "Visualization" + } + ], + "verificationEvidence": [] + }, + { + "id": "NB-PARITY-050", + "name": "Notebook editor persists the latest text after typing stops", + "area": "persistence", + "preconditions": [ + "A disposable notebook with one editable paragraph is open.", + "The user can edit the paragraph." + ], + "action": "Replace the paragraph text and stop typing long enough for the notebook save path to acknowledge the edit.", + "observableOutcomes": [ + { + "id": "NB-PARITY-050-OUTCOME-001", + "description": "The persisted paragraph text equals the latest typed text." + }, + { + "id": "NB-PARITY-050-OUTCOME-002", + "description": "The save assertion is based on observable persistence or wire evidence, not an internal timer." + } + ], + "interpreter": null, + "roleExpectations": { + "owner": "allow", + "writer": "allow", + "reader": "deny", + "runner": "not-applicable" + }, + "roleVerification": { + "owner": "unverified", + "writer": "unverified", + "reader": "unverified", + "runner": "not-applicable" + }, + "coverage": { + "status": "gap", + "tests": [], + "issues": ["ZEPPELIN-6661"], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [ + { + "path": "zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/code-editor/code-editor.component.ts", + "symbol": "NotebookParagraphCodeEditorComponent" + } + ], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts", + "symbol": "NotebookKeyboardPage" + } + ] + }, + { + "id": "NB-PARITY-051", + "name": "Notebook editor does not lose an edit made while a prior save is in flight", + "area": "persistence", + "preconditions": [ + "A disposable notebook with one editable paragraph is open.", + "The first paragraph save request can be observed before it completes." + ], + "action": "Edit the paragraph, keep the first save in flight, then make a second edit.", + "observableOutcomes": [ + { + "id": "NB-PARITY-051-OUTCOME-001", + "description": "The first in-flight save does not overwrite or drop the second edit." + }, + { + "id": "NB-PARITY-051-OUTCOME-002", + "description": "A later observable save or reconciliation persists the second edit." + } + ], + "interpreter": null, + "roleExpectations": { + "owner": "allow", + "writer": "allow", + "reader": "deny", + "runner": "not-applicable" + }, + "roleVerification": { + "owner": "unverified", + "writer": "unverified", + "reader": "unverified", + "runner": "not-applicable" + }, + "coverage": { + "status": "gap", + "tests": [], + "issues": ["ZEPPELIN-6661"], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [ + { + "path": "zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/code-editor/code-editor.component.ts", + "symbol": "NotebookParagraphCodeEditorComponent" + }, + { + "path": "zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.ts", + "symbol": "NotebookComponent" + } + ], + "verificationEvidence": [] + }, + { + "id": "NB-PARITY-060", + "name": "Notebook honors host theme selection", + "area": "theme", + "preconditions": ["The user can choose light, dark, or system theme from the host shell."], + "action": "Change the host theme while a notebook surface is mounted.", + "observableOutcomes": [ + { + "id": "NB-PARITY-060-OUTCOME-001", + "description": "Notebook text remains readable." + }, + { + "id": "NB-PARITY-060-OUTCOME-002", + "description": "Result and chart output inherit the host theme tokens." + }, + { + "id": "NB-PARITY-060-OUTCOME-003", + "description": "The selected theme persists after reload." + } + ], + "interpreter": null, + "coverage": { + "status": "gap", + "tests": [], + "issues": ["ZEPPELIN-6640"], + "uncoveredOutcomes": [] + }, + "implementationEvidence": [ + { + "path": "zeppelin-web-angular/projects/zeppelin-react/src/theme/ZeppelinThemeProvider.tsx", + "symbol": "ZeppelinThemeProvider" + } + ], + "verificationEvidence": [ + { + "path": "zeppelin-web-angular/e2e/tests/theme/dark-mode.spec.ts", + "symbol": "Dark Mode Theme Switching" + } + ] + } + ] +} diff --git a/zeppelin-web-angular/e2e/scenarios/notebook-parity.md b/zeppelin-web-angular/e2e/scenarios/notebook-parity.md new file mode 100644 index 00000000000..3da42b6e890 --- /dev/null +++ b/zeppelin-web-angular/e2e/scenarios/notebook-parity.md @@ -0,0 +1,172 @@ + + +# Notebook Parity Scenarios + + + +Schema: [notebook-parity.schema.json](./notebook-parity.schema.json) + +Scenario/Angular baseline commit: `d5b57b12fd0c5e1d885767aabe06b242debf8300` + +Scope note: This is a prioritized baseline, not a complete Notebook inventory. Before a React vertical slice is declared ready, add every affected behavior to this registry and classify its evidence. + +Coverage note: `covered` mechanically means this registry points to a matching executable Playwright test tag. Semantic adequacy and runtime pass/fail remain review and CI evidence. Role expectations are recorded only when the outcome varies by role, and role verification records whether that expectation has been tested. + +| ID | Area | Scenario | Coverage | Roles | Tests | Issues | +| --- | --- | --- | --- | --- | --- | --- | +| NB-PARITY-001 | navigation | Notebook container structure is visible | covered | not-applicable | zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts
@NB-PARITY-001 | | +| NB-PARITY-002 | navigation | Notebook title can be displayed and edited | covered | owner: allow
writer: allow
reader: deny
runner: not-applicable | zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts
@NB-PARITY-002 | | +| NB-PARITY-003 | editor | Paragraph enters editing mode on double click | covered | owner: allow
writer: allow
reader: deny
runner: not-applicable | zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts
@NB-PARITY-003 | | +| NB-PARITY-004 | editor | Paragraph add buttons are visible | covered | owner: allow
writer: allow
reader: deny
runner: not-applicable | zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts
@NB-PARITY-004 | | +| NB-PARITY-005 | shortcut | Shift+Enter executes a markdown paragraph | covered | owner: allow
writer: allow
reader: deny
runner: allow | zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts
@NB-PARITY-005 | | +| NB-PARITY-010 | editor | History inline completion can be dismissed without losing editor focus | covered | not-applicable | zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts
@NB-PARITY-010 | | +| NB-PARITY-011 | editor | The second Escape after inline completion dismissal blurs the editor | covered | not-applicable | zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts
@NB-PARITY-011 | | +| NB-PARITY-021 | result | Text and table result displays preserve output semantics after paragraph execution | partial | owner: allow
writer: allow
reader: deny
runner: allow | zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts
@NB-PARITY-021 | ZEPPELIN-6514, ZEPPELIN-6516 | +| NB-PARITY-050 | persistence | Notebook editor persists the latest text after typing stops | gap | owner: allow
writer: allow
reader: deny
runner: not-applicable | | ZEPPELIN-6661 | +| NB-PARITY-051 | persistence | Notebook editor does not lose an edit made while a prior save is in flight | gap | owner: allow
writer: allow
reader: deny
runner: not-applicable | | ZEPPELIN-6661 | +| NB-PARITY-060 | theme | Notebook honors host theme selection | gap | not-applicable | | ZEPPELIN-6640 | + +## Scenario Details + +### NB-PARITY-001 Notebook container structure is visible + +- Area: navigation +- Coverage: covered +- Interpreter: not-applicable +- Role verification: not-applicable +- Preconditions: A disposable notebook route is open. +- Action: Render the notebook route. +- Observable outcomes: NB-PARITY-001-OUTCOME-001: The notebook container is visible with the expected container class. +- Implementation evidence: not-applicable +- Verification evidence: zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts (Notebook Container Component); zeppelin-web-angular/e2e/models/notebook-page.ts (NotebookPage) + +### NB-PARITY-002 Notebook title can be displayed and edited + +- Area: navigation +- Coverage: covered +- Interpreter: not-applicable +- Role verification: owner: unverified; writer: unverified; reader: unverified; runner: not-applicable +- Preconditions: A disposable notebook route is open. +- Action: Open the title editor and rename the notebook. +- Observable outcomes: NB-PARITY-002-OUTCOME-001: The title editor is visible and the changed title is reflected in the notebook header. +- Implementation evidence: not-applicable +- Verification evidence: zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts (Notebook Action Bar Functionality); zeppelin-web-angular/e2e/models/notebook-action-bar-page.ts (NotebookActionBarPage) + +### NB-PARITY-003 Paragraph enters editing mode on double click + +- Area: editor +- Coverage: covered +- Interpreter: not-applicable +- Role verification: owner: unverified; writer: unverified; reader: unverified; runner: not-applicable +- Preconditions: A disposable notebook with at least one paragraph is open. +- Action: Double-click the paragraph. +- Observable outcomes: NB-PARITY-003-OUTCOME-001: The code editor becomes visible for the paragraph. +- Implementation evidence: not-applicable +- Verification evidence: zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts (Notebook Paragraph Functionality); zeppelin-web-angular/e2e/models/notebook-paragraph-page.ts (NotebookParagraphPage) + +### NB-PARITY-004 Paragraph add buttons are visible + +- Area: editor +- Coverage: covered +- Interpreter: not-applicable +- Role verification: owner: unverified; writer: unverified; reader: unverified; runner: not-applicable +- Preconditions: A disposable notebook with a paragraph is open. +- Action: Inspect the paragraph controls. +- Observable outcomes: NB-PARITY-004-OUTCOME-001: Add-paragraph controls are visible where a user can add another paragraph. +- Implementation evidence: not-applicable +- Verification evidence: zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts (Notebook Paragraph Functionality); zeppelin-web-angular/e2e/models/notebook-paragraph-page.ts (NotebookParagraphPage) + +### NB-PARITY-005 Shift+Enter executes a markdown paragraph + +- Area: shortcut +- Coverage: covered +- Interpreter: md +- Role verification: owner: unverified; writer: unverified; reader: unverified; runner: unverified +- Preconditions: A disposable notebook paragraph is focused in the code editor. +- Action: Type Markdown content and press Shift+Enter. +- Observable outcomes: NB-PARITY-005-OUTCOME-001: The paragraph executes and renders the Markdown heading result. +- Implementation evidence: not-applicable +- Verification evidence: zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts (ParagraphActions.Run); zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts (NotebookKeyboardPage) + +### NB-PARITY-010 History inline completion can be dismissed without losing editor focus + +- Area: editor +- Coverage: covered +- Interpreter: python +- Role verification: not-applicable +- Preconditions: A notebook contains a Python paragraph with prior text that can seed inline completion. The notebook route is opened with aiInlineComplete enabled. +- Action: Type a completion prefix in Monaco and press Escape while the completion is visible. +- Observable outcomes: NB-PARITY-010-OUTCOME-001: The inline completion suggestion is shown from notebook history. NB-PARITY-010-OUTCOME-002: The Monaco input remains focused after the first Escape dismisses the suggestion. +- Implementation evidence: not-applicable +- Verification evidence: zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts (Inline completion); zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts (NotebookKeyboardPage) + +### NB-PARITY-011 The second Escape after inline completion dismissal blurs the editor + +- Area: editor +- Coverage: covered +- Interpreter: python +- Role verification: not-applicable +- Preconditions: A notebook contains a Python paragraph with prior text that can seed inline completion. The notebook route is opened with aiInlineComplete enabled. The browser is Chromium. +- Action: Press Escape once to dismiss the completion and press Escape again. +- Observable outcomes: NB-PARITY-011-OUTCOME-001: The first Escape keeps Monaco focused. NB-PARITY-011-OUTCOME-002: The second Escape blurs the Monaco input in Chromium. +- Implementation evidence: not-applicable +- Verification evidence: zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts (Inline completion); zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts (NotebookKeyboardPage) + +### NB-PARITY-021 Text and table result displays preserve output semantics after paragraph execution + +- Area: result +- Coverage: partial +- Interpreter: python +- Role verification: owner: unverified; writer: unverified; reader: unverified; runner: unverified +- Preconditions: A notebook has a Python paragraph that prints text. A notebook has a paragraph that returns tabular output. +- Action: Run the paragraph from the paragraph control and inspect the rendered result panel. +- Observable outcomes: NB-PARITY-021-OUTCOME-001: The result display becomes visible and is not empty. NB-PARITY-021-OUTCOME-002: The UI offers every display mode that the Angular notebook exposes for the returned result type. NB-PARITY-021-OUTCOME-003: Visualization control changes preserve the field mapping from result columns to configured dimensions or measures. NB-PARITY-021-OUTCOME-004: The paragraph's persisted config reflects the resulting configuration object after a visualization option changes. NB-PARITY-021-OUTCOME-005: Text and table results expose accessible table output row by row so migrated React rendering can be compared without relying on screenshots alone. +- Implementation evidence: zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/paragraph.component.html (zeppelin-notebook-paragraph-result); zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/progress/progress.component.ts (NotebookParagraphProgressComponent); zeppelin-web-angular/projects/zeppelin-visualization/src/table-transformation.ts (TableTransformation); zeppelin-web-angular/projects/zeppelin-visualization/src/pivot-transformation.ts (PivotTransformation); zeppelin-web-angular/projects/zeppelin-visualization/src/visualization.ts (Visualization) +- Verification evidence: not-applicable +- Uncovered outcomes: NB-PARITY-021-OUTCOME-002: The UI offers every display mode that the Angular notebook exposes for the returned result type. NB-PARITY-021-OUTCOME-003: Visualization control changes preserve the field mapping from result columns to configured dimensions or measures. NB-PARITY-021-OUTCOME-004: The paragraph's persisted config reflects the resulting configuration object after a visualization option changes. NB-PARITY-021-OUTCOME-005: Text and table results expose accessible table output row by row so migrated React rendering can be compared without relying on screenshots alone. + +### NB-PARITY-050 Notebook editor persists the latest text after typing stops + +- Area: persistence +- Coverage: gap +- Interpreter: not-applicable +- Role verification: owner: unverified; writer: unverified; reader: unverified; runner: not-applicable +- Preconditions: A disposable notebook with one editable paragraph is open. The user can edit the paragraph. +- Action: Replace the paragraph text and stop typing long enough for the notebook save path to acknowledge the edit. +- Observable outcomes: NB-PARITY-050-OUTCOME-001: The persisted paragraph text equals the latest typed text. NB-PARITY-050-OUTCOME-002: The save assertion is based on observable persistence or wire evidence, not an internal timer. +- Implementation evidence: zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/code-editor/code-editor.component.ts (NotebookParagraphCodeEditorComponent) +- Verification evidence: zeppelin-web-angular/e2e/models/notebook-keyboard-page.ts (NotebookKeyboardPage) + +### NB-PARITY-051 Notebook editor does not lose an edit made while a prior save is in flight + +- Area: persistence +- Coverage: gap +- Interpreter: not-applicable +- Role verification: owner: unverified; writer: unverified; reader: unverified; runner: not-applicable +- Preconditions: A disposable notebook with one editable paragraph is open. The first paragraph save request can be observed before it completes. +- Action: Edit the paragraph, keep the first save in flight, then make a second edit. +- Observable outcomes: NB-PARITY-051-OUTCOME-001: The first in-flight save does not overwrite or drop the second edit. NB-PARITY-051-OUTCOME-002: A later observable save or reconciliation persists the second edit. +- Implementation evidence: zeppelin-web-angular/src/app/pages/workspace/notebook/paragraph/code-editor/code-editor.component.ts (NotebookParagraphCodeEditorComponent); zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.ts (NotebookComponent) +- Verification evidence: not-applicable + +### NB-PARITY-060 Notebook honors host theme selection + +- Area: theme +- Coverage: gap +- Interpreter: not-applicable +- Role verification: not-applicable +- Preconditions: The user can choose light, dark, or system theme from the host shell. +- Action: Change the host theme while a notebook surface is mounted. +- Observable outcomes: NB-PARITY-060-OUTCOME-001: Notebook text remains readable. NB-PARITY-060-OUTCOME-002: Result and chart output inherit the host theme tokens. NB-PARITY-060-OUTCOME-003: The selected theme persists after reload. +- Implementation evidence: zeppelin-web-angular/projects/zeppelin-react/src/theme/ZeppelinThemeProvider.tsx (ZeppelinThemeProvider) +- Verification evidence: zeppelin-web-angular/e2e/tests/theme/dark-mode.spec.ts (Dark Mode Theme Switching) diff --git a/zeppelin-web-angular/e2e/scenarios/notebook-parity.schema.json b/zeppelin-web-angular/e2e/scenarios/notebook-parity.schema.json new file mode 100644 index 00000000000..6d8155119d2 --- /dev/null +++ b/zeppelin-web-angular/e2e/scenarios/notebook-parity.schema.json @@ -0,0 +1,258 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "notebook-parity.schema.json", + "title": "Notebook parity registry", + "type": "object", + "additionalProperties": false, + "required": ["$schema", "reviewedCommit", "scenarios"], + "properties": { + "$schema": { + "const": "./notebook-parity.schema.json" + }, + "reviewedCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "scenarios": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/scenario" + } + } + }, + "definitions": { + "stringArray": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["path", "symbol"], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "symbol": { + "type": "string", + "minLength": 1 + } + } + }, + "roleExpectations": { + "type": "object", + "additionalProperties": false, + "required": ["owner", "writer", "reader", "runner"], + "properties": { + "owner": { "$ref": "#/definitions/roleExpectation" }, + "writer": { "$ref": "#/definitions/roleExpectation" }, + "reader": { "$ref": "#/definitions/roleExpectation" }, + "runner": { "$ref": "#/definitions/roleExpectation" } + } + }, + "roleExpectation": { + "enum": ["allow", "deny", "not-applicable"] + }, + "roleVerification": { + "type": "object", + "additionalProperties": false, + "required": ["owner", "writer", "reader", "runner"], + "properties": { + "owner": { "$ref": "#/definitions/roleVerificationStatus" }, + "writer": { "$ref": "#/definitions/roleVerificationStatus" }, + "reader": { "$ref": "#/definitions/roleVerificationStatus" }, + "runner": { "$ref": "#/definitions/roleVerificationStatus" } + } + }, + "roleVerificationStatus": { + "enum": ["verified", "unverified", "not-applicable"] + }, + "outcome": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description"], + "properties": { + "id": { + "type": "string", + "pattern": "^NB-PARITY-[0-9]{3}-OUTCOME-[0-9]{3}$" + }, + "description": { + "type": "string", + "minLength": 1 + } + } + }, + "coverageTest": { + "type": "object", + "additionalProperties": false, + "required": ["path", "tag"], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string", + "pattern": "^@NB-PARITY-[0-9]{3}$" + } + } + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["status", "tests", "issues", "uncoveredOutcomes"], + "properties": { + "status": { + "enum": ["covered", "partial", "gap", "blocked"] + }, + "tests": { + "type": "array", + "items": { + "$ref": "#/definitions/coverageTest" + } + }, + "issues": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^ZEPPELIN-[0-9]+$" + } + }, + "uncoveredOutcomes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^NB-PARITY-[0-9]{3}-OUTCOME-[0-9]{3}$" + } + } + }, + "allOf": [ + { + "if": { + "properties": { "status": { "const": "covered" } } + }, + "then": { + "properties": { + "tests": { "type": "array", "minItems": 1 }, + "uncoveredOutcomes": { "type": "array", "maxItems": 0 } + } + } + }, + { + "if": { + "properties": { "status": { "const": "partial" } } + }, + "then": { + "properties": { + "tests": { "type": "array", "minItems": 1 }, + "issues": { "type": "array", "minItems": 1 }, + "uncoveredOutcomes": { "type": "array", "minItems": 1 } + } + } + }, + { + "if": { + "properties": { "status": { "enum": ["gap", "blocked"] } } + }, + "then": { + "properties": { + "tests": { "type": "array", "maxItems": 0 }, + "issues": { "type": "array", "minItems": 1 }, + "uncoveredOutcomes": { "type": "array", "maxItems": 0 } + } + } + } + ] + }, + "scenario": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "area", + "preconditions", + "action", + "observableOutcomes", + "interpreter", + "implementationEvidence", + "verificationEvidence", + "coverage" + ], + "dependencies": { + "roleExpectations": ["roleVerification"], + "roleVerification": ["roleExpectations"] + }, + "anyOf": [ + { + "properties": { + "implementationEvidence": { "type": "array", "minItems": 1 } + } + }, + { + "properties": { + "verificationEvidence": { "type": "array", "minItems": 1 } + } + } + ], + "properties": { + "id": { + "type": "string", + "pattern": "^NB-PARITY-[0-9]{3}$" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "area": { + "enum": [ + "editor", + "execution", + "result", + "visualization", + "shortcut", + "permission", + "collaboration", + "navigation", + "persistence", + "lifecycle", + "theme", + "accessibility" + ] + }, + "preconditions": { "$ref": "#/definitions/stringArray" }, + "action": { + "type": "string", + "minLength": 1 + }, + "observableOutcomes": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/outcome" } + }, + "interpreter": { + "type": ["string", "null"] + }, + "roleExpectations": { "$ref": "#/definitions/roleExpectations" }, + "roleVerification": { "$ref": "#/definitions/roleVerification" }, + "implementationEvidence": { + "type": "array", + "items": { "$ref": "#/definitions/evidence" } + }, + "verificationEvidence": { + "type": "array", + "items": { "$ref": "#/definitions/evidence" } + }, + "coverage": { "$ref": "#/definitions/coverage" } + } + } + } +} diff --git a/zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts b/zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts index 0ef046d3c14..8ae9749bc0f 100644 --- a/zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts +++ b/zeppelin-web-angular/e2e/tests/notebook/action-bar/action-bar-functionality.spec.ts @@ -39,7 +39,7 @@ test.describe('Notebook Action Bar Functionality', () => { await navigateToNotebookWithFallback(page, testNotebook.noteId); }); - test('should display and allow title editing with tooltip', async ({ page }) => { + test('should display and allow title editing with tooltip', { tag: '@NB-PARITY-002' }, async ({ page }) => { const notebookName = `TestNotebook_${Date.now()}`; await expect(actionBarPage.titleEditor).toBeVisible(); diff --git a/zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts b/zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts index 24f70245735..5273e8dfeef 100644 --- a/zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts +++ b/zeppelin-web-angular/e2e/tests/notebook/inline-completion.spec.ts @@ -51,7 +51,7 @@ const openInlineCompletionEditor = async (page: Page) => { test.describe('Inline completion', () => { addPageAnnotationBeforeEach(PAGES.WORKSPACE.NOTEBOOK_PARAGRAPH_CODE_EDITOR); - test('shows history completion and preserves focus when dismissed', async ({ page }) => { + test('shows history completion and preserves focus when dismissed', { tag: '@NB-PARITY-010' }, async ({ page }) => { const { noteId, inputArea } = await openInlineCompletionEditor(page); try { @@ -63,18 +63,22 @@ test.describe('Inline completion', () => { } }); - test('blurs the editor on the second Escape after dismissing completion', async ({ page, browserName }) => { - test.skip(browserName !== 'chromium', 'Monaco handles the second Escape differently in Firefox and WebKit'); - const { noteId, inputArea } = await openInlineCompletionEditor(page); + test( + 'blurs the editor on the second Escape after dismissing completion', + { tag: '@NB-PARITY-011' }, + async ({ page, browserName }) => { + test.skip(browserName !== 'chromium', 'Monaco handles the second Escape differently in Firefox and WebKit'); + const { noteId, inputArea } = await openInlineCompletionEditor(page); - try { - await expect(inputArea).toBeFocused(); - await page.keyboard.press('Escape'); - await expect(inputArea).toBeFocused(); - await page.keyboard.press('Escape'); - await expect(inputArea).not.toBeFocused(); - } finally { - await page.request.delete(`/api/notebook/${noteId}`); + try { + await expect(inputArea).toBeFocused(); + await page.keyboard.press('Escape'); + await expect(inputArea).toBeFocused(); + await page.keyboard.press('Escape'); + await expect(inputArea).not.toBeFocused(); + } finally { + await page.request.delete(`/api/notebook/${noteId}`); + } } - }); + ); }); diff --git a/zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts b/zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts index a6c87814b10..a7ae17a6fe3 100644 --- a/zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts +++ b/zeppelin-web-angular/e2e/tests/notebook/keyboard/notebook-keyboard-shortcuts.spec.ts @@ -67,7 +67,7 @@ test.describe.serial('Comprehensive Keyboard Shortcuts (ShortcutsMap)', () => { // ===== CORE EXECUTION SHORTCUTS ===== test.describe('ParagraphActions.Run: Shift+Enter', () => { - test('should execute markdown paragraph with Shift+Enter', async () => { + test('should execute markdown paragraph with Shift+Enter', { tag: '@NB-PARITY-005' }, async () => { // Given: A paragraph with markdown content await keyboardPage.tryFocusCodeEditor(); await keyboardPage.setCodeEditorContent('%md\n# Test Heading\n\nThis is **bold** text.'); @@ -80,6 +80,7 @@ test.describe.serial('Comprehensive Keyboard Shortcuts (ShortcutsMap)', () => { // waitForParagraphExecution gates on the status text, so it is the assertion and throws if the run never settles. await keyboardPage.waitForParagraphExecution(0); + await expect(keyboardPage.paragraphResult.getByRole('heading', { name: 'Test Heading' })).toBeVisible(); }); }); diff --git a/zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts b/zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts index a8656cc8f39..406e73e8462 100644 --- a/zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts +++ b/zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts @@ -39,7 +39,7 @@ test.describe('Notebook Container Component', () => { await navigateToNotebookWithFallback(page, testNotebook.noteId); }); - test('should display notebook container with proper structure', async () => { + test('should display notebook container with proper structure', { tag: '@NB-PARITY-001' }, async () => { await expect(notebookPage.notebookContainer).toBeVisible(); expect(await notebookPage.getNotebookContainerClass()).toContain('notebook-container'); }); diff --git a/zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts b/zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts index 105c4226202..8a937dc1a16 100644 --- a/zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts +++ b/zeppelin-web-angular/e2e/tests/notebook/paragraph/paragraph-functionality.spec.ts @@ -47,13 +47,13 @@ test.describe('Notebook Paragraph Functionality', () => { await expect(paragraphPage.controlPanel).toBeVisible(); }); - test('should support double-click editing functionality', async () => { + test('should support double-click editing functionality', { tag: '@NB-PARITY-003' }, async () => { await expect(paragraphPage.paragraphContainer).toBeVisible(); await paragraphPage.doubleClickToEdit(); await expect(paragraphPage.codeEditor).toBeVisible(); }); - test('should display add paragraph buttons', async () => { + test('should display add paragraph buttons', { tag: '@NB-PARITY-004' }, async () => { await expect(paragraphPage.addParagraphAbove).toBeVisible(); await expect(paragraphPage.addParagraphAbove).toHaveCount(1); await expect(paragraphPage.addParagraphBelow).toBeVisible(); @@ -66,7 +66,7 @@ test.describe('Notebook Paragraph Functionality', () => { await expect(paragraphPage.runButton).toBeEnabled(); }); - test('should display result system properly', async ({ page }) => { + test('should display result system properly', { tag: '@NB-PARITY-021' }, async ({ page }) => { await expect(page).toHaveURL(/\/notebook\/[^\/]+/, { timeout: 10000 }); await page.waitForLoadState('domcontentloaded'); await expect(paragraphPage.paragraphContainer).toBeVisible({ timeout: 15000 }); diff --git a/zeppelin-web-angular/package-lock.json b/zeppelin-web-angular/package-lock.json index 68ed893fabc..2fcf37e2562 100644 --- a/zeppelin-web-angular/package-lock.json +++ b/zeppelin-web-angular/package-lock.json @@ -62,6 +62,7 @@ "@types/webpack-env": "^1.18.8", "@vitest/coverage-v8": "4.1.8", "@vitest/eslint-plugin": "^1.6.27", + "ajv": "^8.20.0", "angular-eslint": "21.4.0", "concurrently": "9.2.1", "cross-env": "^10.1.0", @@ -1342,6 +1343,23 @@ } } }, + "node_modules/@angular-devkit/core/node_modules/ajv": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/@angular-devkit/schematics": { "version": "21.2.13", "dev": true, @@ -8213,7 +8231,9 @@ } }, "node_modules/ajv": { - "version": "8.18.0", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", "dependencies": { diff --git a/zeppelin-web-angular/package.json b/zeppelin-web-angular/package.json index f40fadc5e8c..2e9a509bb28 100644 --- a/zeppelin-web-angular/package.json +++ b/zeppelin-web-angular/package.json @@ -14,6 +14,8 @@ "build:projects": "npm run build-project:sdk && npm run build-project:vis", "build-project:sdk": "ng build --project zeppelin-sdk", "check:websocket-contract": "node --test scripts/check-websocket-contract.test.js && node scripts/check-websocket-contract.js", + "generate:notebook-parity-scenarios": "node scripts/generate-notebook-parity-scenarios.mjs", + "check:notebook-parity-scenarios": "node --test scripts/check-notebook-parity-scenarios.test.mjs && node scripts/check-notebook-parity-scenarios.mjs", "build-project:vis": "ng build --project zeppelin-visualization", "lint": "cross-env NODE_OPTIONS='--max-old-space-size=8192' ng lint && npm run lint:react && prettier --check \"**/*.{ts,tsx,mts,js,json,css,html}\"", "lint:fix": "cross-env NODE_OPTIONS='--max-old-space-size=8192' ng lint --fix && npm run lint:fix:react && prettier --write \"**/*.{ts,tsx,mts,js,json,css,html}\"", @@ -91,6 +93,7 @@ "@types/webpack-env": "^1.18.8", "@vitest/coverage-v8": "4.1.8", "@vitest/eslint-plugin": "^1.6.27", + "ajv": "^8.20.0", "angular-eslint": "21.4.0", "concurrently": "9.2.1", "cross-env": "^10.1.0", diff --git a/zeppelin-web-angular/scripts/check-notebook-parity-scenarios.mjs b/zeppelin-web-angular/scripts/check-notebook-parity-scenarios.mjs new file mode 100644 index 00000000000..cd650bebb97 --- /dev/null +++ b/zeppelin-web-angular/scripts/check-notebook-parity-scenarios.mjs @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { loadRegistry, markdownPath, renderMarkdown, validateRegistry, webRoot } from './notebook-parity-scenarios.mjs'; + +const registry = loadRegistry(webRoot); +const errors = validateRegistry(registry, webRoot, { checkMarkdown: false }); +const expectedMarkdown = renderMarkdown(registry); +const actualMarkdownPath = path.join(webRoot, markdownPath); + +if (!existsSync(actualMarkdownPath)) { + errors.push(`${markdownPath} does not exist`); +} else { + const actualMarkdown = readFileSync(actualMarkdownPath, 'utf8'); + if (actualMarkdown !== expectedMarkdown) { + errors.push(`${markdownPath} is stale; run npm run generate:notebook-parity-scenarios`); + } +} + +if (errors.length > 0) { + console.error(errors.join('\n')); + process.exit(1); +} diff --git a/zeppelin-web-angular/scripts/check-notebook-parity-scenarios.test.mjs b/zeppelin-web-angular/scripts/check-notebook-parity-scenarios.test.mjs new file mode 100644 index 00000000000..fd66c915e62 --- /dev/null +++ b/zeppelin-web-angular/scripts/check-notebook-parity-scenarios.test.mjs @@ -0,0 +1,351 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { spawnSync } from 'node:child_process'; + +import { renderMarkdown, validateRegistry } from './notebook-parity-scenarios.mjs'; + +function createFixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'notebook-parity-')); + const webRoot = path.join(root, 'zeppelin-web-angular'); + fs.mkdirSync(path.join(root, 'zeppelin-web-angular/e2e/tests/notebook/main'), { recursive: true }); + fs.mkdirSync(path.join(root, 'zeppelin-web-angular/e2e/scenarios'), { recursive: true }); + fs.mkdirSync(path.join(root, 'e2e/scenarios'), { recursive: true }); + fs.mkdirSync(path.join(root, 'zeppelin-web-angular/src/app/pages/workspace/notebook'), { recursive: true }); + fs.writeFileSync( + path.join(root, 'zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.ts'), + 'class NotebookComponent {}' + ); + fs.writeFileSync( + path.join(root, 'zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts'), + "import { test } from '@playwright/test';\ntest('should render', { tag: '@NB-PARITY-001' }, async () => {});" + ); + spawnSync('git', ['init'], { cwd: root, stdio: 'ignore' }); + spawnSync('git', ['config', 'user.email', 'test@example.invalid'], { cwd: root, stdio: 'ignore' }); + spawnSync('git', ['config', 'user.name', 'Test'], { cwd: root, stdio: 'ignore' }); + spawnSync('git', ['add', '.'], { cwd: root, stdio: 'ignore' }); + spawnSync('git', ['commit', '-m', 'fixture'], { cwd: root, stdio: 'ignore' }); + const commit = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(); + return { commit, root, webRoot }; +} + +function writeFixtureSpec(webRoot, source) { + fs.writeFileSync(path.join(webRoot, 'e2e/tests/notebook/main/notebook-container.spec.ts'), source); +} + +function baseRegistry(commit) { + return { + $schema: './notebook-parity.schema.json', + reviewedCommit: commit, + scenarios: [ + { + id: 'NB-PARITY-001', + name: 'Container renders', + area: 'navigation', + preconditions: ['note exists'], + action: 'open the route', + observableOutcomes: [{ id: 'NB-PARITY-001-OUTCOME-001', description: 'container is visible' }], + interpreter: null, + roleExpectations: { + owner: 'allow', + writer: 'allow', + reader: 'allow', + runner: 'allow' + }, + roleVerification: { + owner: 'unverified', + writer: 'unverified', + reader: 'unverified', + runner: 'unverified' + }, + implementationEvidence: [ + { + path: 'zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.ts', + symbol: 'NotebookComponent' + } + ], + verificationEvidence: [], + coverage: { + status: 'covered', + tests: [ + { + path: 'zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts', + tag: '@NB-PARITY-001' + } + ], + issues: [], + uncoveredOutcomes: [] + } + } + ] + }; +} + +test('validates a current registry and generated markdown', () => { + const { commit, root, webRoot } = createFixture(); + const registry = baseRegistry(commit); + fs.writeFileSync(path.join(webRoot, 'e2e/scenarios/notebook-parity.md'), renderMarkdown(registry)); + + assert.deepEqual(validateRegistry(registry, webRoot), []); +}); + +test('accepts multiline executable Playwright test declarations', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + writeFixtureSpec( + webRoot, + `import { test } from '@playwright/test'; + test( + 'should render', + { tag: '@NB-PARITY-001' }, + async () => {} + );` + ); + fs.writeFileSync(path.join(webRoot, 'e2e/scenarios/notebook-parity.md'), renderMarkdown(registry)); + + assert.deepEqual(validateRegistry(registry, webRoot), []); +}); + +test('accepts an aliased Playwright test import', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + writeFixtureSpec( + webRoot, + "import { test as pwTest } from '@playwright/test';\npwTest('should render', { tag: '@NB-PARITY-001' }, async () => {});" + ); + + assert.deepEqual(validateRegistry(registry, webRoot, { checkMarkdown: false }), []); +}); + +test('rejects duplicate ids', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios.push(structuredClone(registry.scenarios[0])); + + const errors = validateRegistry(registry, webRoot, { checkMarkdown: false }).join('\n'); + assert.match(errors, /is duplicated/); +}); + +test('rejects stale generated markdown', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + fs.writeFileSync(path.join(webRoot, 'e2e/scenarios/notebook-parity.md'), 'stale\n'); + + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /notebook-parity\.md is stale/); +}); + +test('rejects false covered claims without matching Playwright tag', () => { + const { commit, root, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].coverage.tests[0].tag = '@NB-PARITY-999'; + fs.writeFileSync(path.join(webRoot, 'e2e/scenarios/notebook-parity.md'), renderMarkdown(registry)); + + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /tag must be @NB-PARITY-001/); +}); + +test('rejects commented-only, helper-string, and skipped Playwright coverage claims', () => { + const cases = [ + { + name: 'commented-only', + source: "// test('should render', { tag: '@NB-PARITY-001' }, async () => {});" + }, + { + name: 'helper-string', + source: "const parityTag = '@NB-PARITY-001';\ntest('should render', { tag: parityTag }, async () => {});" + }, + { + name: 'test.skip', + source: + "import { test } from '@playwright/test';\ntest.skip('should render', { tag: '@NB-PARITY-001' }, async () => {});" + }, + { + name: 'skipped describe', + source: + "import { test } from '@playwright/test';\ntest.describe.skip('disabled', () => { test('should render', { tag: '@NB-PARITY-001' }, async () => {}); });" + }, + { + name: 'non-Playwright test helper', + source: "const test = () => undefined;\ntest('should render', { tag: '@NB-PARITY-001' }, async () => {});" + } + ]; + + for (const { name, source } of cases) { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + writeFixtureSpec(webRoot, source); + fs.writeFileSync(path.join(webRoot, 'e2e/scenarios/notebook-parity.md'), renderMarkdown(registry)); + + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /tag is not declared by an executable test\(\)/, name); + } +}); + +test('requires Jira issues for gaps', () => { + const { commit, root, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].coverage.status = 'gap'; + registry.scenarios[0].coverage.tests = []; + fs.writeFileSync(path.join(webRoot, 'e2e/scenarios/notebook-parity.md'), renderMarkdown(registry)); + + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /coverage\.issues must contain at least 1 item/); +}); + +test('requires executable coverage, a Jira issue, and named uncovered outcomes for partial scenarios', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].coverage.status = 'partial'; + registry.scenarios[0].coverage.issues = []; + + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /coverage\.issues must contain at least 1 item/); + + registry.scenarios[0].coverage.issues = ['ZEPPELIN-1234']; + registry.scenarios[0].coverage.tests = []; + const missingTestErrors = validateRegistry(registry, webRoot).join('\n'); + assert.match(missingTestErrors, /coverage\.tests must contain at least 1 item/); + assert.match(missingTestErrors, /coverage\.uncoveredOutcomes must contain at least 1 item/); +}); + +test('rejects invalid Jira issue keys and extra role expectation fields', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].coverage.issues = ['OTHER-123']; + registry.scenarios[0].roleExpectations.admin = 'allow'; + + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /must match \^ZEPPELIN-\[0-9\]\+\$/); + assert.match(errors, /roleExpectations\.admin is not allowed/); +}); + +test('rejects malformed coverage arrays without applying unsafe coverage rules', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].coverage.tests = null; + registry.scenarios[0].coverage.issues = null; + registry.scenarios[0].coverage.uncoveredOutcomes = null; + + assert.doesNotThrow(() => validateRegistry(registry, webRoot)); + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /coverage\.tests must be array/); + assert.match(errors, /coverage\.issues must be array/); + assert.match(errors, /coverage\.uncoveredOutcomes must be array/); +}); + +test('rejects malformed test entries without throwing', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].coverage.tests = [null]; + + assert.doesNotThrow(() => validateRegistry(registry, webRoot)); + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /coverage\.tests\.0 must be object/); + + registry.scenarios[0].coverage.tests = [ + { + path: 'zeppelin-web-angular/e2e/tests/notebook/main/notebook-container.spec.ts', + tag: '@NB-PARITY-001', + unexpected: true + } + ]; + const metadataErrors = validateRegistry(registry, webRoot).join('\n'); + assert.match(metadataErrors, /coverage\.tests\.0\.unexpected is not allowed/); +}); + +test('allows verified role evidence and optional role metadata', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].roleVerification.reader = 'verified'; + assert.deepEqual(validateRegistry(registry, webRoot, { checkMarkdown: false }), []); + + delete registry.scenarios[0].roleExpectations; + delete registry.scenarios[0].roleVerification; + assert.deepEqual(validateRegistry(registry, webRoot, { checkMarkdown: false }), []); +}); + +test('rejects not-applicable verification for an applicable role expectation', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].roleVerification.reader = 'not-applicable'; + + const errors = validateRegistry(registry, webRoot, { checkMarkdown: false }).join('\n'); + assert.match(errors, /roleVerification\.reader must not be not-applicable/); +}); + +test('rejects applicable verification for a not-applicable role expectation', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].roleExpectations.reader = 'not-applicable'; + registry.scenarios[0].roleVerification.reader = 'verified'; + + const errors = validateRegistry(registry, webRoot, { checkMarkdown: false }).join('\n'); + assert.match(errors, /roleVerification\.reader must be not-applicable/); +}); + +test('rejects a null registry without throwing', () => { + assert.doesNotThrow(() => validateRegistry(null, undefined, { checkMarkdown: false })); + assert.match(validateRegistry(null, undefined, { checkMarkdown: false }).join('\n'), /registry must be object/); +}); + +test('rejects implementation evidence paths outside the repository', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].implementationEvidence[0].path = '../../outside-repository.ts'; + + const errors = validateRegistry(registry, webRoot).join('\n'); + assert.match(errors, /implementationEvidence\[0\].path does not exist/); +}); + +test('references uncovered outcomes by stable id', () => { + const { commit, webRoot } = createFixture(); + const registry = baseRegistry(commit); + registry.scenarios[0].coverage.status = 'partial'; + registry.scenarios[0].coverage.issues = ['ZEPPELIN-1234']; + registry.scenarios[0].coverage.uncoveredOutcomes = ['NB-PARITY-001-OUTCOME-999']; + + const errors = validateRegistry(registry, webRoot, { checkMarkdown: false }).join('\n'); + assert.match(errors, /must reference an observable outcome id/); +}); + +test('renders generated Markdown with exactly one trailing newline', () => { + const { commit } = createFixture(); + const markdown = renderMarkdown(baseRegistry(commit)); + + assert.equal(markdown.endsWith('\n'), true); + assert.equal(markdown.endsWith('\n\n'), false); +}); + +test('renders generated Markdown with an Apache License header', () => { + const { commit } = createFixture(); + const markdown = renderMarkdown(baseRegistry(commit)); + + assert.match(markdown, /^', + '', + '# Notebook Parity Scenarios', + '', + '', + '', + 'Schema: [notebook-parity.schema.json](./notebook-parity.schema.json)', + '', + `Scenario/Angular baseline commit: \`${registry.reviewedCommit}\``, + '', + 'Scope note: This is a prioritized baseline, not a complete Notebook inventory. Before a React vertical slice is declared ready, add every affected behavior to this registry and classify its evidence.', + '', + 'Coverage note: `covered` mechanically means this registry points to a matching executable Playwright test tag. Semantic adequacy and runtime pass/fail remain review and CI evidence. Role expectations are recorded only when the outcome varies by role, and role verification records whether that expectation has been tested.', + '', + '| ID | Area | Scenario | Coverage | Roles | Tests | Issues |', + '| --- | --- | --- | --- | --- | --- | --- |' + ]; + + for (const scenario of registry.scenarios) { + const rolesText = scenario.roleExpectations + ? roles.map(role => `${role}: ${scenario.roleExpectations[role]}`).join('
') + : 'not-applicable'; + const testsText = + scenario.coverage.tests.length === 0 + ? '' + : scenario.coverage.tests.map(test => `${test.path}
${test.tag}`).join('

'); + const issuesText = scenario.coverage.issues.join(', '); + lines.push( + `| ${scenario.id} | ${scenario.area} | ${escapeTableCell(scenario.name)} | ${scenario.coverage.status} | ${escapeTableCell(rolesText)} | ${escapeTableCell(testsText)} | ${issuesText} |` + ); + } + + lines.push('', '## Scenario Details', ''); + + for (const scenario of registry.scenarios) { + lines.push(`### ${scenario.id} ${scenario.name}`); + lines.push(''); + lines.push(`- Area: ${scenario.area}`); + lines.push(`- Coverage: ${scenario.coverage.status}`); + lines.push(`- Interpreter: ${scenario.interpreter ?? 'not-applicable'}`); + lines.push( + `- Role verification: ${scenario.roleVerification ? roles.map(role => `${role}: ${scenario.roleVerification[role]}`).join('; ') : 'not-applicable'}` + ); + lines.push(`- Preconditions: ${scenario.preconditions.join(' ')}`); + lines.push(`- Action: ${scenario.action}`); + lines.push( + `- Observable outcomes: ${scenario.observableOutcomes.map(outcome => `${outcome.id}: ${outcome.description}`).join(' ')}` + ); + lines.push( + `- Implementation evidence: ${scenario.implementationEvidence.map(item => `${item.path} (${item.symbol})`).join('; ') || 'not-applicable'}` + ); + lines.push( + `- Verification evidence: ${scenario.verificationEvidence.map(item => `${item.path} (${item.symbol})`).join('; ') || 'not-applicable'}` + ); + if (scenario.coverage.uncoveredOutcomes.length > 0) { + const outcomesById = new Map(scenario.observableOutcomes.map(outcome => [outcome.id, outcome.description])); + lines.push( + `- Uncovered outcomes: ${scenario.coverage.uncoveredOutcomes.map(id => `${id}: ${outcomesById.get(id)}`).join(' ')}` + ); + } + lines.push(''); + } + + return `${lines.join('\n').replace(/\n+$/, '')}\n`; +}; + +export const validateRegistry = (registry, root = webRoot, { checkMarkdown = true } = {}) => { + const errors = []; + + if (!validateSchema(registry)) { + errors.push(...validateSchema.errors.map(formatSchemaError)); + } + + if (!registry || typeof registry !== 'object' || Array.isArray(registry)) { + return errors; + } + + if (typeof registry.reviewedCommit === 'string' && /^[0-9a-f]{40}$/.test(registry.reviewedCommit)) { + try { + execFileSync('git', ['cat-file', '-e', `${registry.reviewedCommit}^{commit}`], { cwd: root, stdio: 'ignore' }); + } catch { + errors.push(`reviewedCommit is not available in this checkout: ${registry.reviewedCommit}`); + } + } + if (!Array.isArray(registry.scenarios) || registry.scenarios.length === 0) { + return errors; + } + + const ids = new Set(); + let previousId = ''; + for (const [index, scenario] of registry.scenarios.entries()) { + const prefix = `scenarios[${index}]`; + if (!scenario || typeof scenario !== 'object') { + continue; + } + if (typeof scenario.id === 'string' && ids.has(scenario.id)) { + errors.push(`${prefix}.id is duplicated: ${scenario.id}`); + } else if (typeof scenario.id === 'string') { + if (previousId && scenario.id <= previousId) { + errors.push(`${prefix}.id must sort after ${previousId}`); + } + previousId = scenario.id; + ids.add(scenario.id); + } + + if (scenario.roleExpectations && scenario.roleVerification) { + for (const role of roles) { + if ( + scenario.roleExpectations[role] === 'not-applicable' && + scenario.roleVerification[role] !== 'not-applicable' + ) { + errors.push(`${prefix}.roleVerification.${role} must be not-applicable`); + } else if ( + scenario.roleExpectations[role] !== 'not-applicable' && + scenario.roleVerification[role] === 'not-applicable' + ) { + errors.push( + `${prefix}.roleVerification.${role} must not be not-applicable when roleExpectations.${role} is ${scenario.roleExpectations[role]}` + ); + } + } + } + + for (const evidenceField of ['implementationEvidence', 'verificationEvidence']) { + if (Array.isArray(scenario[evidenceField])) { + for (const [evidenceIndex, evidence] of scenario[evidenceField].entries()) { + if (!evidence || typeof evidence.path !== 'string') { + continue; + } + const evidencePath = resolveRepositoryPath(root, evidence.path); + if (!evidencePath || !existsSync(evidencePath)) { + errors.push(`${prefix}.${evidenceField}[${evidenceIndex}].path does not exist: ${evidence.path}`); + } + } + } + } + + const outcomeIds = new Set(); + if (Array.isArray(scenario.observableOutcomes)) { + for (const [outcomeIndex, outcome] of scenario.observableOutcomes.entries()) { + if (!outcome || typeof outcome.id !== 'string') { + continue; + } + if (!outcome.id.startsWith(`${scenario.id}-OUTCOME-`)) { + errors.push(`${prefix}.observableOutcomes[${outcomeIndex}].id must start with ${scenario.id}-OUTCOME-`); + } else if (outcomeIds.has(outcome.id)) { + errors.push(`${prefix}.observableOutcomes[${outcomeIndex}].id is duplicated: ${outcome.id}`); + } + outcomeIds.add(outcome.id); + } + } + + const coverage = scenario.coverage; + if (!coverage || typeof coverage !== 'object') { + continue; + } + const uncoveredOutcomes = Array.isArray(coverage.uncoveredOutcomes) ? coverage.uncoveredOutcomes : []; + for (const [outcomeIndex, outcomeId] of uncoveredOutcomes.entries()) { + if (!outcomeIds.has(outcomeId)) { + errors.push(`${prefix}.coverage.uncoveredOutcomes[${outcomeIndex}] must reference an observable outcome id`); + } + } + const coverageTests = Array.isArray(coverage.tests) ? coverage.tests : []; + for (const [testIndex, test] of coverageTests.entries()) { + if (!test || typeof test !== 'object') { + continue; + } + if (typeof test.path === 'string') { + const testPath = resolveRepositoryPath(root, test.path); + if (!testPath || !existsSync(testPath)) { + errors.push(`${prefix}.coverage.tests[${testIndex}].path does not exist: ${test.path}`); + } + if (coverage.status === 'covered' && !test.path.startsWith('zeppelin-web-angular/e2e/tests/notebook/')) { + errors.push(`${prefix}.coverage.tests[${testIndex}].path must be in the notebook E2E suite`); + } + } + if (test.tag !== `@${scenario.id}`) { + errors.push(`${prefix}.coverage.tests[${testIndex}].tag must be @${scenario.id}`); + } else if (typeof test.path === 'string' && !testDeclaresExecutableTag(root, test)) { + errors.push( + `${prefix}.coverage.tests[${testIndex}].tag is not declared by an executable test() in ${test.path}` + ); + } + } + } + + if (checkMarkdown && errors.length === 0) { + const actualMarkdownPath = path.join(root, markdownPath); + if (!existsSync(actualMarkdownPath)) { + errors.push(`${markdownPath} does not exist`); + } else if (readFileSync(actualMarkdownPath, 'utf8') !== renderMarkdown(registry)) { + errors.push(`${markdownPath} is stale; run npm run generate:notebook-parity-scenarios`); + } + } + + return errors; +}; + +export const loadRegistry = (root = webRoot) => readJson(path.join(root, registryPath));