fix(tests): reserve the run name, hold Run for unsaved edits, cancel judges on close

- A test named "run" collided with the static run endpoint, so its detail
  page got a 405; the name is now reserved.
- Run is disabled while the open editor holds edits the server has not
  saved (including a refused save), so a run never uses the previous source.
- toMatchRubric model calls are aborted when the sandbox run ends, so a
  stopped test no longer keeps calling or billing the judge.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EHGBgHrtePpi7KMrEfy41R

Author: TheodoreSpeaks

apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import { useState } from 'react'
+import { useEffect, useState } from 'react'
 import { toast } from '@sim/emcn'
 import { Columns2, Eye, Pencil, PlayOutline, ShieldCheck } from '@sim/emcn/icons'
 import { getErrorMessage } from '@sim/utils/errors'
@@ -19,6 +19,7 @@ import {
   useWorkflowTest,
 } from '@/hooks/queries/workflow-tests'
 import { useAddressedWorkspaceFileRecord } from '@/hooks/queries/workspace-files'
+import { testRunSelectionKey, useTestRunSelectionStore } from '@/stores/workflow-tests/store'
 
 interface TestTarget {
   workspaceId: string
@@ -74,12 +75,15 @@ interface TestRunActionInput extends TestTarget {
 export function useTestRunAction({ workspaceId, name, running }: TestRunActionInput) {
   const canEdit = useUserPermissionsContext().canEdit === true
   const runTests = useRunWorkflowTests(workspaceId)
+  const unsaved = useTestRunSelectionStore(
+    (state) => state.unsavedTests[testRunSelectionKey(workspaceId, name)] === true
+  )
   const busy = running || runTests.isPending
   return {
     id: 'run',
     icon: PlayOutline,
-    text: busy ? 'Running…' : 'Run',
-    disabled: !canEdit || busy,
+    text: busy ? 'Running…' : unsaved ? 'Unsaved' : 'Run',
+    disabled: !canEdit || busy || unsaved,
     onSelect: () =>
       runTests.mutate(
         { version: VERSION, names: [name] },
@@ -107,6 +111,9 @@ export function TestView({ workspaceId, name, previewMode }: TestViewProps) {
   const query = useWorkflowTest(workspaceId, name)
   const refreshTest = useRefreshWorkflowTest(workspaceId, name)
   const [saveError, setSaveError] = useState<string | null>(null)
+  const setUnsaved = useTestRunSelectionStore((state) => state.setUnsaved)
+  const testKey = testRunSelectionKey(workspaceId, name)
+  useEffect(() => () => setUnsaved(testKey, false), [setUnsaved, testKey])
   const detail = query.data
   const file = useAddressedWorkspaceFileRecord(workspaceId, detail?.test.fileId ?? '', {
     enabled: Boolean(detail),
@@ -142,6 +149,7 @@ export function TestView({ workspaceId, name, previewMode }: TestViewProps) {
         preview={<TestDashboard workspaceId={workspaceId} name={name} detail={detail} />}
         sourceSide='end'
         enableFind
+        onDirtyChange={(dirty) => setUnsaved(testKey, dirty)}
         onSaveError={(saveFailure) =>
           setSaveError(getErrorMessage(saveFailure, 'The test file was not saved'))
         }

apps/sim/lib/api/contracts/mothership-tests.ts

@@ -1,15 +1,15 @@
 import { z } from 'zod'
 
 const scope = z.object({ workspaceId: z.string().min(1).max(100).optional() })
-/** A test file name: `tests/<name>.test.js`. */
-export const TEST_NAME_PATTERN = /^[a-z0-9][a-z0-9-]{0,79}$/
+/** A test file name: `tests/<name>.test.js`. `run` is taken by the run endpoint beside the names. */
+export const TEST_NAME_PATTERN = /^(?!run$)[a-z0-9][a-z0-9-]{0,79}$/
 
 const nameSchema = z
   .string()
   .trim()
   .regex(
     TEST_NAME_PATTERN,
-    'Use a lowercase name of letters, numbers, and dashes, like billing-routing'
+    'Use a lowercase name of letters, numbers, and dashes, like billing-routing (not "run")'
   )
   .describe('Test file name: tests/<name>.test.js, lowercase letters, numbers, and dashes.')
 const versionSchema = z

apps/sim/lib/workflow-tests/judge.ts

@@ -32,6 +32,7 @@ export interface JudgeRubricInput {
   actorUserId: string
   value: string
   rubric: string
+  abortSignal: AbortSignal
 }
 
 /** One pass/fail model call for `toMatchRubric`, billed to the workspace like a wand call. */
@@ -65,6 +66,7 @@ export async function judgeRubric(input: JudgeRubricInput): Promise<JudgeVerdict
     },
     temperature: 0,
     workspaceId: input.workspaceId,
+    abortSignal: input.abortSignal,
   })
   if (!('content' in response) || typeof response.content !== 'string') {
     throw new Error('The judge did not return a verdict')

apps/sim/lib/workflow-tests/paths.ts

@@ -24,7 +24,7 @@ export function assertTestName(name: string): void {
   if (!TEST_NAME_PATTERN.test(name)) {
     throw new OrchestrationError(
       'validation',
-      `"${testFilePath(name)}" is not a valid test file. Use tests/<name>.test.js with a lowercase name of letters, numbers, and dashes, like tests/billing-routing.test.js`
+      `"${testFilePath(name)}" is not a valid test file. Use tests/<name>.test.js with a lowercase name of letters, numbers, and dashes other than "run", like tests/billing-routing.test.js`
     )
   }
 }

apps/sim/lib/workflow-tests/session.ts

@@ -524,6 +524,8 @@ export class WorkflowTestSession {
   readonly enteredWorkflowIds = new Set<string>()
   /** Set when the sandbox run ends; a start still awaiting its lookups then starts nothing. */
   private closed = false
+  /** Stops `toMatchRubric` model calls still in flight when the sandbox run ends. */
+  private readonly judgeAbort = new AbortController()
 
   constructor(readonly config: WorkflowTestSessionConfig) {}
 
@@ -586,6 +588,7 @@ export class WorkflowTestSession {
 
   close(): void {
     this.closed = true
+    this.judgeAbort.abort(new Error('The test run ended'))
     this.cancel([...this.runs.keys()])
   }
 
@@ -595,6 +598,7 @@ export class WorkflowTestSession {
       actorUserId: requirePrincipalSubjectUserId(this.config.principal),
       value: args.value,
       rubric: args.rubric,
+      abortSignal: this.judgeAbort.signal,
     })
   }
 

apps/sim/stores/workflow-tests/store.ts

@@ -8,20 +8,29 @@ import { devtools } from 'zustand/middleware'
  */
 interface TestRunSelectionState {
   selectedRunIds: Record<string, string>
+  /** Tests whose open editor holds edits the server has not saved; Run waits for them. */
+  unsavedTests: Record<string, true>
   selectRun: (testKey: string, runId: string | null) => void
+  setUnsaved: (testKey: string, unsaved: boolean) => void
   reset: () => void
 }
 
 export const useTestRunSelectionStore = create<TestRunSelectionState>()(
   devtools(
     (set) => ({
       selectedRunIds: {},
+      unsavedTests: {},
       selectRun: (testKey, runId) =>
         set((state) => {
           const { [testKey]: _previous, ...rest } = state.selectedRunIds
           return { selectedRunIds: runId === null ? rest : { ...rest, [testKey]: runId } }
         }),
-      reset: () => set({ selectedRunIds: {} }),
+      setUnsaved: (testKey, unsaved) =>
+        set((state) => {
+          const { [testKey]: _previous, ...rest } = state.unsavedTests
+          return { unsavedTests: unsaved ? { ...rest, [testKey]: true } : rest }
+        }),
+      reset: () => set({ selectedRunIds: {}, unsavedTests: {} }),
     }),
     { name: 'test-run-selection' }
   )