Correct E2E teardown ordering and remove browser gates

packages/e2e/playwright.config.ts

@@ -31,6 +31,7 @@ export default defineConfig({
         'tests/smoke-pty.spec.ts',
         'tests/fixture-toml.spec.ts',
         'tests/auth-diagnostics.spec.ts',
+        'tests/app-management-api.spec.ts',
       ],
     },
     {
@@ -45,6 +46,7 @@ export default defineConfig({
         'tests/smoke-pty.spec.ts',
         'tests/fixture-toml.spec.ts',
         'tests/auth-diagnostics.spec.ts',
+        'tests/app-management-api.spec.ts',
       ],
       dependencies: ['remote-auth'],
     },

packages/e2e/setup/app-management-api.ts

@@ -0,0 +1,79 @@
+/* eslint-disable no-restricted-imports -- this wrapper runs the cli-kit API client in a tsx subprocess */
+import {execa} from 'execa'
+import * as path from 'path'
+import {fileURLToPath} from 'url'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const RESULT_PREFIX = 'E2E_APP_MANAGEMENT_RESULT='
+
+export type AppDeletionReadiness =
+  | {status: 'ready'; app: {id: string; key: string}}
+  | {status: 'already-deleted'}
+  | {status: 'still-installed'; installCount: number}
+
+export interface AppManagementAppState {
+  id: string
+  key: string
+  installCount?: number | null
+  activeRelease: {version: {name: string}}
+}
+
+interface AppDeletionReadinessOptions {
+  appName: string
+  clientId?: string
+  orgId: string
+}
+
+/**
+ * Inspect the app and wait for its install count to reach zero.
+ *
+ * The subprocess is required because Playwright's transformed module graph
+ * cannot load cli-kit's dist ESM on every supported CI Node version. Running
+ * the API work under tsx also lets teardown use cli-kit's normal GraphQL
+ * throttling, network retry, and token-refresh behavior.
+ */
+export async function waitForAppDeletionReadiness(
+  sessionEnv: NodeJS.ProcessEnv,
+  options: AppDeletionReadinessOptions,
+): Promise<AppDeletionReadiness> {
+  const script = path.join(__dirname, 'inspect-app-management-state.ts')
+  const result = await execa('tsx', [script], {
+    env: {...sessionEnv, SHOPIFY_FLAG_VERBOSE: undefined},
+    extendEnv: false,
+    preferLocal: true,
+    localDir: path.resolve(__dirname, '..'),
+    input: JSON.stringify(options),
+    timeout: 120_000,
+  })
+
+  const resultLine = result.stdout.split('\n').findLast((line) => line.startsWith(RESULT_PREFIX))
+  if (!resultLine) {
+    throw new Error('App Management inspection did not return a result')
+  }
+
+  return JSON.parse(resultLine.slice(RESULT_PREFIX.length)) as AppDeletionReadiness
+}
+
+export function appDeletionReadinessFromApps(
+  apps: AppManagementAppState[],
+  appName: string,
+  clientId?: string,
+): AppDeletionReadiness {
+  const exactNameMatches = apps.filter((app) => app.activeRelease.version.name === appName)
+  const clientIdMatches = clientId ? exactNameMatches.filter((app) => app.key === clientId) : []
+  const matchingApps = clientIdMatches.length > 0 ? clientIdMatches : exactNameMatches
+
+  if (matchingApps.length === 0) return {status: 'already-deleted'}
+  if (matchingApps.length > 1) {
+    throw new Error(`App Management API returned multiple apps named ${appName}`)
+  }
+
+  const app = matchingApps[0]!
+  if (typeof app.installCount !== 'number') {
+    throw new Error(`App Management API did not return installCount for ${appName}`)
+  }
+
+  return app.installCount === 0
+    ? {status: 'ready', app: {id: app.id, key: app.key}}
+    : {status: 'still-installed', installCount: app.installCount}
+}

packages/e2e/setup/app.ts

@@ -1,6 +1,6 @@
-/* eslint-disable no-restricted-imports, no-await-in-loop */
+/* eslint-disable no-restricted-imports */
 import {authFixture} from './auth.js'
-import {getLastPageStatus, isVisibleWithin, navigateToDashboard, refreshIfPageError} from './browser.js'
+import {getLastPageStatus, isVisibleWithin} from './browser.js'
 import {CLI_TIMEOUT, BROWSER_TIMEOUT} from './constants.js'
 import * as toml from '@iarna/toml'
 import * as path from 'path'
@@ -320,53 +320,52 @@ export async function configLink(
 }
 
 // ---------------------------------------------------------------------------
-// Dev dashboard browser actions — find and delete apps
+// Dev dashboard browser actions — delete apps
 // ---------------------------------------------------------------------------
 
-/** Search dev dashboard for an app by name. Returns the app URL or null. */
-export async function findAppOnDevDashboard(page: Page, appName: string, orgId?: string): Promise<string | null> {
-  const org = orgId ?? (process.env.E2E_ORG_ID ?? '').trim()
-  const email = process.env.E2E_ACCOUNT_EMAIL
+/**
+ * Load the app's settings page, clicking through the accounts.shopify.com
+ * account picker if the session bounces there — which is common when the
+ * browser context has not visited the Dev Dashboard yet.
+ */
+async function gotoAppSettings(page: Page, appUrl: string): Promise<void> {
+  await page.goto(`${appUrl}/settings`, {waitUntil: 'domcontentloaded'})
+  await page.waitForTimeout(BROWSER_TIMEOUT.medium)
 
-  await navigateToDashboard({browserPage: page, email, orgId: org})
+  if (!isAccountsShopifyUrl(page.url())) return
 
-  // Scan current page + pagination for the app
-  while (true) {
-    const allLinks = await page.locator('a[href*="/apps/"]').all()
-    for (const link of allLinks) {
-      const text = (await link.textContent()) ?? ''
-      if (text.includes(appName)) {
-        const href = await link.getAttribute('href')
-        if (href) return href.startsWith('http') ? href : `https://dev.shopify.com${href}`
-      }
+  const email = process.env.E2E_ACCOUNT_EMAIL
+  if (email) {
+    const accountButton = page.locator(`text=${email}`).first()
+    if (await isVisibleWithin(accountButton, BROWSER_TIMEOUT.long)) {
+      await accountButton.click()
+      await page.waitForTimeout(BROWSER_TIMEOUT.medium)
     }
-
-    // Check for next page
-    const nextLink = page.locator('a[href*="next_cursor"]').first()
-    if (!(await isVisibleWithin(nextLink, BROWSER_TIMEOUT.medium))) break
-    const nextHref = await nextLink.getAttribute('href')
-    if (!nextHref) break
-    const nextUrl = nextHref.startsWith('http') ? nextHref : `https://dev.shopify.com${nextHref}`
-    await page.goto(nextUrl, {waitUntil: 'domcontentloaded'})
-    await page.waitForTimeout(BROWSER_TIMEOUT.medium)
-    await refreshIfPageError(page)
   }
+  await page.goto(`${appUrl}/settings`, {waitUntil: 'domcontentloaded'})
+  await page.waitForTimeout(BROWSER_TIMEOUT.medium)
+}
 
-  return null
+function isAccountsShopifyUrl(rawUrl: string): boolean {
+  try {
+    return new URL(rawUrl).hostname === 'accounts.shopify.com'
+    // eslint-disable-next-line no-catch-all/no-catch-all
+  } catch {
+    return false
+  }
 }
 
 /**
  * Delete an app from its dev dashboard settings page. Returns true if deleted.
  *
  * Single attempt — caller owns the retry loop.
  *
- * Fail-fast on STILL_HAS_INSTALLS: the Delete button stays disabled while
- * installs exist, so we throw to let the caller skip instead of spinning.
+ * Throws STILL_HAS_INSTALLS when the Delete button remains disabled after a
+ * reload so the caller can apply its bounded retry policy.
  */
 export async function deleteAppFromDevDashboard(page: Page, appUrl: string): Promise<boolean> {
   // Step 1: Navigate to the app's settings page. 404 → already deleted. 5xx → throw for retry.
-  await page.goto(`${appUrl}/settings`, {waitUntil: 'domcontentloaded'})
-  await page.waitForTimeout(BROWSER_TIMEOUT.medium)
+  await gotoAppSettings(page, appUrl)
   const gotoStatus = getLastPageStatus(page)
   if (gotoStatus === 404) return true
   if (gotoStatus !== undefined && gotoStatus >= 500) {
@@ -377,6 +376,11 @@ export async function deleteAppFromDevDashboard(page: Page, appUrl: string): Pro
   // Button can be below the fold, and takes ~1-2s to enable after uninstall (one reload covers propagation lag).
   // If it stays disabled after reload, installs remain — fail fast for caller.
   const deleteBtn = page.locator('button:has-text("Delete app")').first()
+  if (!(await isVisibleWithin(deleteBtn, BROWSER_TIMEOUT.long))) {
+    // Include the landed URL: the usual cause is a session bounce that the
+    // account-picker handling above did not cover.
+    throw new Error(`Delete app button not found (page: ${page.url()})`)
+  }
   await deleteBtn.scrollIntoViewIfNeeded({timeout: BROWSER_TIMEOUT.long})
   if (!(await deleteBtn.isEnabled())) {
     await page.reload({waitUntil: 'domcontentloaded'})

packages/e2e/setup/inspect-app-management-state.ts

@@ -0,0 +1,106 @@
+/* eslint-disable @nx/enforce-module-boundaries, no-await-in-loop -- this
+   subprocess uses cli-kit's built API client with the isolated E2E session */
+import {appDeletionReadinessFromApps} from './app-management-api.js'
+import {appManagementFqdn} from '../../cli-kit/dist/public/node/context/fqdn.js'
+import {graphqlRequest} from '../../cli-kit/dist/public/node/api/graphql.js'
+import {ensureAuthenticatedAppManagementAndBusinessPlatform} from '../../cli-kit/dist/public/node/session.js'
+import {loadtestHeaderRecord} from '../helpers/loadtest-header.js'
+import type {AppDeletionReadiness, AppManagementAppState} from './app-management-api.js'
+
+const RESULT_PREFIX = 'E2E_APP_MANAGEMENT_RESULT='
+
+interface InspectionOptions {
+  appName: string
+  clientId?: string
+  orgId: string
+}
+
+interface AppsQueryResult {
+  appsConnection?: {edges: {node: AppManagementAppState}[]} | null
+}
+
+const query = `
+  query E2ETeardownApps($query: String) {
+    appsConnection(query: $query, first: 50) {
+      edges {
+        node {
+          id
+          key
+          installCount
+          activeRelease {
+            version {
+              name
+            }
+          }
+        }
+      }
+    }
+  }
+`
+
+const options = JSON.parse(await readStandardInput()) as InspectionOptions
+const deadline = Date.now() + 30_000
+const missingAppConfirmationsRequired = 2
+const apiUrl = `https://${await appManagementFqdn()}/app_management/unstable/graphql.json`
+
+let {appManagementToken} = await ensureAuthenticatedAppManagementAndBusinessPlatform({noPrompt: true})
+
+async function inspectApp(): Promise<AppDeletionReadiness> {
+  const result = await graphqlRequest<AppsQueryResult>({
+    api: 'App Management',
+    url: apiUrl,
+    token: appManagementToken,
+    addedHeaders: loadtestHeaderRecord(),
+    query,
+    variables: {
+      query: `title:${options.appName}`,
+      // App Management reads this undeclared variable to route the request to the organization.
+      organizationId: options.orgId,
+    },
+    unauthorizedHandler: {
+      type: 'token_refresh',
+      handler: async () => {
+        const refreshed = await ensureAuthenticatedAppManagementAndBusinessPlatform({
+          noPrompt: true,
+          forceRefresh: true,
+        })
+        appManagementToken = refreshed.appManagementToken
+        return {token: appManagementToken}
+      },
+    },
+  })
+
+  if (!result.appsConnection) {
+    throw new Error('App Management API did not return appsConnection')
+  }
+
+  return appDeletionReadinessFromApps(
+    result.appsConnection.edges.map((edge) => edge.node),
+    options.appName,
+    options.clientId,
+  )
+}
+
+let readiness: AppDeletionReadiness
+let missingAppConfirmations = 0
+
+while (true) {
+  readiness = await inspectApp()
+  missingAppConfirmations = readiness.status === 'already-deleted' ? missingAppConfirmations + 1 : 0
+
+  if (readiness.status === 'ready') break
+  if (readiness.status === 'already-deleted' && missingAppConfirmations >= missingAppConfirmationsRequired) break
+  if (Date.now() >= deadline) break
+
+  await new Promise((resolve) => setTimeout(resolve, 2_000))
+}
+
+process.stdout.write(`\n${RESULT_PREFIX}${JSON.stringify(readiness)}\n`)
+
+async function readStandardInput(): Promise<string> {
+  let input = ''
+  for await (const chunk of process.stdin) {
+    input += chunk.toString()
+  }
+  return input
+}

packages/e2e/setup/store.ts

@@ -162,7 +162,7 @@ export async function uninstallAppFromStore(page: Page, storeSlug: string, appNa
 
     // Force a DOM click to bypass Playwright actionability (the button can read as
     // disabled mid-transition).
-    await confirmBtn.evaluate((button) => button.click())
+    await confirmBtn.evaluate((button) => (button as HTMLButtonElement).click())
     await page.waitForTimeout(BROWSER_TIMEOUT.medium)
   }