diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 3c6e5c05..6ad4a907 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -23,6 +23,7 @@ import { isFalse, isTurboScale, shouldAddServiceVersion, + getTestPlanId, } from '../util.js' import PerformanceTester from '../instrumentation/performance/performance-tester.js' import { EVENTS as PerformanceEvents } from '../instrumentation/performance/constants.js' @@ -32,7 +33,6 @@ import type { Options, Capabilities } from '@wdio/types' import type { BrowserstackConfig, BrowserstackOptions, - TestManagementOptions, TestObservabilityOptions, } from '../types.js' import { TestFrameworkConstants } from './frameworks/constants/testFrameworkConstants.js' @@ -47,7 +47,7 @@ const CLI_DOWNLOAD_TMP_SUFFIX = '.zip' export class CLIUtils { static automationFrameworkDetail = {} static testFrameworkDetail = {} - static CLISupportedFrameworks = ['mocha', 'cucumber'] + static CLISupportedFrameworks = ['mocha', 'cucumber', 'jasmine'] static isDevelopmentEnv() { return process.env.BROWSERSTACK_CLI_ENV === 'development' @@ -116,11 +116,7 @@ export class CLIUtils { ) const observabilityOptions: TestObservabilityOptions = options.testObservabilityOptions || {} - const testManagementOptions: TestManagementOptions = - options.testManagementOptions || {} - const testPlanId = typeof testManagementOptions.testPlanId === 'string' - ? testManagementOptions.testPlanId.trim() - : '' + const testPlanId = (getTestPlanId(options as BrowserstackConfig & Options.Testrunner) || '').trim() const binconfig: Record = { userName: observabilityOptions.user || config.user, accessKey: observabilityOptions.key || config.key, diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts new file mode 100644 index 00000000..ff4041ab --- /dev/null +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -0,0 +1,496 @@ +import path from 'node:path' +import util from 'node:util' +import { v4 as uuidv4 } from 'uuid' +import type { Frameworks } from '@wdio/types' +import type { HookStats, TestStats } from '@wdio/reporter' + +import TestFramework from './testFramework.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' +import { HookState } from '../states/hookState.js' +import TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import TrackedInstance from '../instances/trackedInstance.js' +import { CLIUtils } from '../cliUtils.js' +import { TestFrameworkConstants } from './constants/testFrameworkConstants.js' +import { BStackLogger as logger } from '../cliLogger.js' +import { getHookType, getTestTags, removeAnsiColors } from '../../util.js' +import { shouldProcessEventForTesthub } from '../../testHub/utils.js' +import { TEST_ANALYTICS_ID } from '../../constants.js' +import type TestHubModule from '../modules/testHubModule.js' + +/** What the reporter knows at each event and the framework cannot see: its suite stack and suite file. */ +export interface JasmineSuiteContext { + scopes: string[] + suiteFile?: string +} + +interface FailureFields { + failure?: Array<{ backtrace: string[] }> + failureReason?: string + failureType?: string | null +} + +/** + * CLI test framework for `framework: 'jasmine'` under WebdriverIO. + * + * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), + * hook taxonomy and skip/exclude states differ from mocha's. + * + * Two event sources, as on the legacy flow, both through `trackEvent`: + * - the WDIO reporter (`reporter.ts`, `source: 'reporter'`) feeds Test Observability only — it is the one + * jasmine source that sees pending, focused-out and beforeAll-failed specs. Its hooks are not awaited by + * WDIO, so its events are applied through one ordered queue and sent straight to TestHubModule, never to + * the other modules; + * - the service's awaited beforeTest/afterTest drive the product modules (a11y, Automate, Percy) through + * the observers, for executed specs only, with TestHub skipped. + */ +export default class WdioJasmineTestFramework extends TestFramework { + // Same data keys WdioMochaTestFramework uses: TestHubModule's LOG arm reads them through + // WdioMochaTestFramework.getLogEntries/clearLogs. + static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' + static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + static KEY_TEST_RETRIES = 'test_retries' + static KEY_TEST_DURATION = 'test_duration' + static KEY_HOOK_IDENTIFIER = 'hook_identifier' + static KEY_HOOK_SCOPE = 'hook_scope' + static KEY_HOOK_SCOPES = 'hook_scopes' + static KEY_HOOK_DURATION = 'hook_duration' + static KEY_HOOK_FAILURE = 'hook_failure' + static KEY_HOOK_FAILURE_TYPE = 'hook_failure_type' + static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' + + static #hookTypes = new Map([ + ['beforeAll', 'BEFORE_ALL'], + ['afterAll', 'AFTER_ALL'], + ['beforeEach', 'BEFORE_EACH'], + ['afterEach', 'AFTER_EACH'], + ]) + + #specInstances = new Map() + // Open specs by fullName: how the service's hooks find the instance the reporter minted. + #specsByFullName = new Map() + // Every spec of this worker by uuid: legacy attaches HTTP logs and screenshots to the last started spec, even after it ended. + #specsByUuid = new Map() + #serviceOnly = new Set() + #suiteTitles = new Map() + #testHub: TestHubModule | null = null + // beforeAll/afterAll never overlap, so a hook's uid (its title) is unique while it is open. + #hookInstances = new Map() + #openHook: TestFrameworkInstance | null = null + #lastSpec: TestFrameworkInstance | null = null + #queue: Promise = Promise.resolve() + #pendingEvents = 0 + + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { + super(testFrameworks, testFrameworkVersions, binSessionId) + logger.debug('WdioJasmineTestFramework: constructed') + } + + /** True when no reporter event is still waiting to be dispatched (the reporter's `isSynchronised`). */ + isIdle() { + return this.#pendingEvents === 0 + } + + /** The WDIO `hookName` a jasmine hook runs under, as the hook-type key `getHookType` returns for mocha titles. */ + static hookTypeFromName(hookName: string | undefined) { + return WdioJasmineTestFramework.#hookTypes.get(hookName ?? '') ?? 'unknown' + } + + /** The state the reporter reports a hook in: BEFORE_ALL/AFTER_ALL, else NONE (each-hooks are not reported). */ + static reporterHookState(hookTitle: string | undefined): State { + return WdioJasmineTestFramework.#hookState(getHookType(String(hookTitle ?? '').toLowerCase())) ?? TestFrameworkState.NONE + } + + setTestHubModule(testHub: TestHubModule | null | undefined) { + this.#testHub = testHub ?? null + } + + /** + * Reporter events (`source: 'reporter'`) are applied before this returns its promise: TEST PRE writes the + * spec's uuid to `args.testUuid`. The service's hooks call this without a source: INIT_TEST pins the + * spec's instance and uuid; TEST PRE/POST reach the product modules. LOG_REPORT and the hook states carry + * nothing jasmine needs: results come from the reporter. + */ + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + if (args.source === 'reporter') { + this.#trackReporterEvent(testFrameworkState, hookState, args) + return + } + if (testFrameworkState === TestFrameworkState.LOG) { + this.#log(args.logEntry as Record) + return + } + try { + await this.#queue + if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { + return + } + const test = (args.test ?? {}) as { fullName?: string, description?: string, file?: string } + const instance = this.#serviceInstance(test) + TestFramework.setTrackedInstance(instance.getContext(), instance) + process.env[TEST_ANALYTICS_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) + if (testFrameworkState === TestFrameworkState.INIT_TEST) { + return + } + + const moduleArgs: Record = { ...args } + if (hookState === HookState.PRE) { + this.#suiteTitles.set(instance, args.suiteTitle) + } else { + // afterTest hands over the raw `Jasmine__TopLevel__Suite`; the modules need the describe chain beforeTest derived + moduleArgs.suiteTitle = this.#suiteTitles.get(instance) ?? args.suiteTitle + this.#suiteTitles.delete(instance) + if (this.#serviceOnly.delete(instance) && test.fullName && this.#specsByFullName.get(test.fullName) === instance) { + this.#specsByFullName.delete(test.fullName) + } + } + await this.#toModules(instance, testFrameworkState, hookState, moduleArgs) + } catch (error) { + logger.error(`WdioJasmineTestFramework: service event ${testFrameworkState}/${hookState} failed: ${util.format(error)}`) + } + } + + #trackReporterEvent(testFrameworkState: State, hookState: State, args: Record) { + const context = args.context as JasmineSuiteContext + if (testFrameworkState === TestFrameworkState.TEST) { + if (hookState === HookState.PRE) { + args.testUuid = this.#testStarted(args.testStats as TestStats, context) + } else { + this.#testEnded(args.testStats as TestStats, context) + } + } else if (testFrameworkState === TestFrameworkState.LOG) { + this.#log(args.logEntry as Record | undefined) + } else if (hookState === HookState.PRE) { + this.#hookStarted(testFrameworkState, args.hookStats as HookStats, context) + } else { + this.#hookEnded(testFrameworkState, args.hookStats as HookStats) + } + } + + #testStarted(testStats: TestStats, context: JasmineSuiteContext) { + try { + logger.debug(`WdioJasmineTestFramework: test start uid=${testStats.uid}`) + const instance = this.#createInstance() + const scopes = [...context.scopes] + const fullTitle = testStats.fullTitle + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + [TestFrameworkConstants.KEY_TEST_ID]: fullTitle, + [TestFrameworkConstants.KEY_TEST_NAME]: testStats.title, + [TestFrameworkConstants.KEY_TEST_SCOPE]: fullTitle, + [TestFrameworkConstants.KEY_TEST_SCOPES]: scopes, + [TestFrameworkConstants.KEY_TEST_TAGS]: getTestTags(testStats as unknown as Frameworks.Test, scopes), + ...this.#filePaths(context.suiteFile), + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: testStats.start?.toISOString(), + [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, + [WdioJasmineTestFramework.KEY_TEST_RETRIES]: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, + }) + this.#specInstances.set(testStats.uid, instance) + this.#specsByFullName.set(fullTitle, instance) + this.#specsByUuid.set(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string, instance) + // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. + TestFramework.setTrackedInstance(instance.getContext(), instance) + + const args = { test: this.#specArg(testStats, context), suiteTitle: this.#suiteTitle(testStats) } + this.#enqueue('TEST/PRE', async () => { + this.#lastSpec = instance + await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.PRE, args) + }) + return TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string + } catch (error) { + logger.error(`WdioJasmineTestFramework: test start failed: ${util.format(error)}`) + } + } + + #testEnded(testStats: TestStats, context: JasmineSuiteContext) { + try { + const instance = this.#specInstances.get(testStats.uid) + if (!instance) { + logger.debug(`WdioJasmineTestFramework: no instance for test end uid=${testStats.uid}`) + return + } + this.#specInstances.delete(testStats.uid) + if (this.#specsByFullName.get(testStats.fullTitle) === instance) { + this.#specsByFullName.delete(testStats.fullTitle) + } + + const state = testStats.state + const error = testStats.error + let result: string = state + let failure: FailureFields = {} + if (state === 'failed') { + result = (error && error.message && error.message.includes('sync skip; aborting execution')) ? 'ignore' : 'failed' + if (error) { + failure = this.#failureFields(error) + } + } + const entries: Record = { + [TestFrameworkConstants.KEY_TEST_RESULT]: result, + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: testStats.start?.toISOString(), + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: testStats.end?.toISOString(), + [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), + } + if (failure.failure) { + entries[TestFrameworkConstants.KEY_TEST_FAILURE] = failure.failure + entries[TestFrameworkConstants.KEY_TEST_FAILURE_REASON] = failure.failureReason + entries[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE] = failure.failureType + } + const testResult = { + passed: state === 'passed', + skipped: state === 'skipped', + error, + duration: testStats._duration, + retries: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, + } + const args = { test: this.#specArg(testStats, context), result: testResult, suiteTitle: this.#suiteTitle(testStats) } + this.#enqueue('TEST/POST', async () => { + instance.updateMultipleEntries(entries) + await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.POST, args) + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: test end failed: ${util.format(error)}`) + } + } + + #hookStarted(hookFrameworkState: State, hookStats: HookStats, context: JasmineSuiteContext) { + try { + const title = hookStats.title + const key = getHookType(String(title ?? '').toLowerCase()) + if (!WdioJasmineTestFramework.#hookState(key) || context.scopes.length === 0) { + logger.debug(`WdioJasmineTestFramework: hook start not reported title=${title} scopes=${context.scopes.length}`) + return + } + const scopes = [...context.scopes] + const instance = this.#createInstance() + const hookId = uuidv4() + const hook: Record = { + key, + [TestFrameworkConstants.KEY_HOOK_ID]: hookId, + [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: hookStats.start?.toISOString(), + [TestFrameworkConstants.KEY_HOOK_LOGS]: [], + [TestFrameworkConstants.KEY_HOOK_NAME]: title, + [WdioJasmineTestFramework.KEY_HOOK_IDENTIFIER]: `${title} for ${scopes[scopes.length - 1]}`, + [WdioJasmineTestFramework.KEY_HOOK_SCOPE]: `${scopes[0]} - ${title}`, + [WdioJasmineTestFramework.KEY_HOOK_SCOPES]: scopes, + [WdioJasmineTestFramework.KEY_HOOK_DURATION]: hookStats._duration, + ...this.#filePaths(context.suiteFile), + } + instance.updateData(TestFrameworkConstants.KEY_HOOK_ID, hookId) + // The session event is keyed by KEY_TEST_UUID: it links the hook run to its Automate session and platform + instance.updateData(TestFrameworkConstants.KEY_TEST_UUID, hookId) + this.#hookInstances.set(hookStats.uid, instance) + + const emitHook = shouldProcessEventForTesthub('HookRunStarted') + this.#enqueue(`${key}/PRE`, async () => { + this.#openHook = instance + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map + hooksStarted.set(key, [hook]) + instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_STARTED, key) + if (emitHook) { + await this.#toTestHub(instance, hookFrameworkState, HookState.PRE, { hook: hookStats }) + } + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: hook start failed: ${util.format(error)}`) + } + } + + #hookEnded(hookFrameworkState: State, hookStats: HookStats) { + try { + const instance = this.#hookInstances.get(hookStats.uid) + if (!instance) { + return + } + this.#hookInstances.delete(hookStats.uid) + const key = getHookType(String(hookStats.title ?? '').toLowerCase()) + + const state = hookStats.state + const error = hookStats.error + let result = state as string | undefined + let failure: FailureFields = {} + if (state === 'failed') { + result = (error && error.message && error.message.includes('sync skip; aborting execution')) ? 'ignore' : 'failed' + if (error) { + failure = this.#failureFields(error) + } + } + const endedAt = hookStats.end?.toISOString() + const duration = hookStats._duration + const emitHook = shouldProcessEventForTesthub('HookRunFinished') + this.#enqueue(`${key}/POST`, async () => { + if (this.#openHook === instance) { + this.#openHook = null + } + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map[]> + const hooksFinished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map[]> + const hook = hooksStarted.get(key)?.pop() + if (!hook) { + return + } + if (result) { + hook[TestFrameworkConstants.KEY_HOOK_RESULT] = result + } + hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = endedAt + hook[WdioJasmineTestFramework.KEY_HOOK_DURATION] = duration + if (failure.failure) { + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE] = failure.failure + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE_REASON] = failure.failureReason + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE_TYPE] = failure.failureType + } + hooksFinished.set(key, [hook]) + instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_FINISHED, key) + if (emitHook) { + await this.#toTestHub(instance, hookFrameworkState, HookState.POST, { hook: hookStats, result: { passed: result === 'passed', error } }) + } + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: hook end failed: ${util.format(error)}`) + } + } + + /** + * Console logs: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. + * HTTP command logs and screenshots name their spec (`test_run_uuid`), as legacy did; unknown uuids are dropped. + */ + #log(logEntry: Record | undefined) { + try { + if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { + return + } + const targetUuid = logEntry.test_run_uuid as string | undefined + this.#enqueue('LOG/POST', async () => { + const instance = targetUuid ? this.#specsByUuid.get(targetUuid) : (this.#openHook ?? this.#lastSpec) + if (!instance) { + return + } + const logRecord: Record = { + kind: logEntry.kind ?? TestFrameworkConstants.KIND_LOG, + message: Buffer.from(String(logEntry.message ?? '')), + level: logEntry.level, + timestamp: logEntry.timestamp, + } + if (instance === this.#openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) + } + // Sent in the instance's own state, not LOG: the binary keys an entry `hook_run_uuid` only when + // its state is a hook state, so an open all-hook's logs land on the hook. + if (this.#testHub) { + await this.#testHub.sendLogCreatedEvent({ instance, logEntries: [logRecord] }) + } + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: log failed: ${util.format(error)}`) + } + } + + #createInstance() { + const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + TestFrameworkState.NONE, + HookState.NONE + ) + const frameworkName = this.getTestFrameworks()[0] + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + }) + return instance + } + + async #toTestHub(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + const testHub = this.#testHub + if (!testHub) { + logger.debug(`WdioJasmineTestFramework: TestHub module not loaded; ${testFrameworkState}/${hookState} not reported`) + return + } + this.updateInstanceState(instance, testFrameworkState, hookState) + args.instance = instance + // Legacy sent a run's platform and session with its start event, hooks included (parity #19, #23) + if (hookState === HookState.PRE) { + testHub.onBeforeTest(args) + } + await testHub.sendTestFrameworkEvent(args) + } + + async #toModules(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + args.instance = instance + args.skipTestHub = true + await this.runHooks(instance, testFrameworkState, hookState, args) + } + + /** The reporter's instance for this spec; minted here only when the reporter is not feeding (TO opted out). */ + #serviceInstance(test: { fullName?: string, description?: string, file?: string }) { + const existing = test.fullName ? this.#specsByFullName.get(test.fullName) : undefined + if (existing) { + return existing + } + const instance = this.#createInstance() + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + [TestFrameworkConstants.KEY_TEST_ID]: test.fullName, + [TestFrameworkConstants.KEY_TEST_NAME]: test.description, + ...this.#filePaths(test.file), + }) + this.#serviceOnly.add(instance) + if (test.fullName) { + this.#specsByFullName.set(test.fullName, instance) + } + return instance + } + + #enqueue(label: string, step: () => Promise) { + this.#pendingEvents++ + this.#queue = this.#queue.then(step).catch((error) => { + logger.error(`WdioJasmineTestFramework: ${label} failed: ${util.format(error)}`) + }).finally(() => { + this.#pendingEvents-- + }) + } + + /** The binary re-bases `test_file_path` itself, so it must be absolute. */ + #filePaths(suiteFile: string | undefined) { + const absolute = suiteFile ? path.resolve(suiteFile) : undefined + return { + [TestFrameworkConstants.KEY_TEST_FILE_PATH]: absolute, + [TestFrameworkConstants.KEY_TEST_LOCATION]: absolute ? path.relative(process.cwd(), absolute) : undefined, + } + } + + #failureFields(error: Error): FailureFields { + return { + failure: [{ backtrace: [removeAnsiColors(error.message), removeAnsiColors(error.stack || '')] }], + failureReason: removeAnsiColors(error.message), + failureType: error.message === null ? null : error.message.toString().match(/AssertionError/) ? 'AssertionError' : 'UnhandledError', + } + } + + /** SpecResult-shaped, like the service hooks' `test`: jasmine has no `title`. */ + #specArg(testStats: TestStats, context: JasmineSuiteContext) { + return { description: testStats.title, fullName: testStats.fullTitle, file: context.suiteFile } as unknown as Frameworks.Test + } + + /** The describe chain, derived the way `service.beforeTest` does for jasmine. */ + #suiteTitle(testStats: TestStats) { + const fullName = testStats.fullTitle || '' + return fullName.slice(0, fullName.indexOf(testStats.title || '') - 1) + } + + static #hookState(key: string) { + if (key === 'BEFORE_ALL') { + return TestFrameworkState.BEFORE_ALL + } + if (key === 'AFTER_ALL') { + return TestFrameworkState.AFTER_ALL + } + return null + } +} diff --git a/packages/browserstack-service/src/cli/grpcClient.ts b/packages/browserstack-service/src/cli/grpcClient.ts index 2aabc1cd..909dd160 100644 --- a/packages/browserstack-service/src/cli/grpcClient.ts +++ b/packages/browserstack-service/src/cli/grpcClient.ts @@ -55,6 +55,27 @@ const SUMMARY_ANSI = { warn: { base: '\x1b[33m', emphasis: '\x1b[1;33m' } } +const TEST_PLAN_ID_ARG = '--browserstack.testManagementOptions.testPlanId' + +// The binary parses CLI args as `--key=value` only, so the space form reaches it as `true` and +// overrides the id getBinConfig already resolved; the id travels in the bin config instead. +export function stripTestPlanIdArg(args: string[]): string[] { + const result: string[] = [] + for (let i = 0; i < args.length; i++) { + if (args[i] === TEST_PLAN_ID_ARG) { + if (i + 1 < args.length && !args[i + 1].startsWith('-')) { + i++ + } + continue + } + if (args[i].startsWith(`${TEST_PLAN_ID_ARG}=`)) { + continue + } + result.push(args[i]) + } + return result +} + /** * GrpcClient - Singleton class for managing gRPC client connections * @@ -180,7 +201,7 @@ export class GrpcClient { sdkVersion: packageVersion, pathProject: process.cwd(), pathConfig: path.resolve(process.cwd(), 'browserstack.yml'), - cliArgs: process.argv.slice(2), + cliArgs: stripTestPlanIdArg(process.argv.slice(2)), frameworks: [automationFrameworkDetail.name, testFrameworkDetail.name], frameworkVersions, language: CLIUtils.getSdkLanguage(), diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 444ec6d7..56eaf74c 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -17,6 +17,7 @@ import type { Options } from '@wdio/types' import TestOpsConfig from '../testOps/testOpsConfig.js' import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from './frameworks/wdioJasmineTestFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' @@ -48,7 +49,7 @@ export class BrowserstackCLI { modulesLoaded = false binSessionId: string | null = null modules: Record = {} - testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|null = null + testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|WdioJasmineTestFramework|null = null cliParams: Record | null = null automationFramework: WdioAutomationFramework|null = null SDK_CLI_BIN_PATH: string | null = null @@ -178,7 +179,11 @@ export class BrowserstackCLI { this.modules[ObservabilityModule.MODULE_NAME] = new ObservabilityModule(startBinResponse.observability) } - this.modules[TestHubModule.MODULE_NAME] = new TestHubModule(startBinResponse.testhub) + const testHubModule = new TestHubModule(startBinResponse.testhub) + this.modules[TestHubModule.MODULE_NAME] = testHubModule + if (this.testFramework instanceof WdioJasmineTestFramework) { + this.testFramework.setTestHubModule(testHubModule) + } // Custom-tag (multi Test-Case-ID) tagging rides the per-test event_json // to TestHub, so it is gated on the testhub pipeline being active. @@ -555,6 +560,10 @@ export class BrowserstackCLI { this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) return } + if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-jasmine') { + this.testFramework = new WdioJasmineTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + return + } // An unmatched name leaves testFramework null, and every CLI event then no-ops with no // error of any kind. Name it so the silence is diagnosable. this.logger.error(`setupTestFramework: no CLI test framework registered for name=${testFrameworkDetail.name}; test events will not be tracked`) diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index edcb1b3b..d8cf2a26 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -278,7 +278,7 @@ export default class AccessibilityModule extends BaseModule { // on suite and test titles, and in this window neither exists yet. onBeforeTest // re-computes the per-test gate, tags included, so this only affects the window. const preTestSessionId = this.currentSessionId() - if (this.autoScanning && preTestSessionId !== undefined && preTestSessionId !== null) { + if (this.autoScanning && this.supportsPreTestWindow() && preTestSessionId !== undefined && preTestSessionId !== null) { this.accessibilityMap.set(preTestSessionId, true) this.logger.debug('Accessibility scan gate opened ahead of the first test') } @@ -338,9 +338,11 @@ export default class AccessibilityModule extends BaseModule { try { this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.') const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' - const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string, description?: string } : {}) || {} + // jasmine specs carry their name in `description` (`title` is unset), as on the classic handler + const testTitle = test.title ?? test.description - this.currentTestName = test.title || null + this.currentTestName = testTitle || null this.testContextSeen = true const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() @@ -353,7 +355,7 @@ export default class AccessibilityModule extends BaseModule { // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg // behaviour — both extra args arrive undefined/false and the tag branch is not taken. const world = args.world as { [key: string]: unknown } | undefined - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility + const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, testTitle || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility this.accessibilityMap.set(sessionId, shouldScanTest) @@ -537,6 +539,12 @@ export default class AccessibilityModule extends BaseModule { return false } + // Legacy jasmine opened no scan window before its first spec (classic PRE_TEST_SCAN_FRAMEWORKS) + private supportsPreTestWindow(): boolean { + const frameworks = BrowserstackCLI.getInstance().getTestFramework()?.getTestFrameworks() ?? [] + return !frameworks.some((name) => String(name).toLowerCase().includes('jasmine')) + } + // See the classic handler: one definition, used at every scan site. private get hasNoParent(): boolean { return !this.currentHookRunUuid && !this.testContextSeen diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index d01d3080..7da8dbcd 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -1,4 +1,5 @@ import BaseModule from './baseModule.js' +import { BrowserstackCLI } from '../index.js' import { BStackLogger } from '../cliLogger.js' import TestFramework from '../frameworks/testFramework.js' import { TestFrameworkState } from '../states/testFrameworkState.js' @@ -23,6 +24,20 @@ interface TestResult { reason?: string } +/** What legacy `service.after()` computed the jasmine session verdict from, as the service tracked it. */ +interface SessionVerdictInputs { + result: number + specsRan: boolean + failReasons: string[] + pureTestFailReasons: string[] + hookFailReasons: string[] +} + +interface SessionVerdict { + status: 'passed' | 'failed' + reason?: string +} + interface SessionData { lastTestName: string appliedName?: string // last name successfully PUT for this session, for de-duping @@ -74,6 +89,11 @@ export default class AutomateModule extends BaseModule { return } + // Legacy annotated from service.beforeTest, which never runs for cucumber or a skipped test + if (!this.isCucumberInstance(instace) && args.skipReport !== true) { + await this.annotate(browser, `Test: ${test.fullName ?? test.title}`) + } + // `setSessionName: false` suppresses the NAME, not the registration. The session still has // to enter sessionMap or onAfterExecute has nothing to status-mark, and legacy marks it // either way — its `after()` status block gates on setSessionStatus alone. Registering with @@ -88,9 +108,10 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + const sessionNameFormat = this.sessionNameFormatFor(instace, testContextOptions) + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -181,9 +202,10 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + const sessionNameFormat = this.sessionNameFormatFor(instace, testContextOptions) + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -324,6 +346,30 @@ export default class AutomateModule extends BaseModule { } } + /** + * The binary's config echo cannot carry a function, so `sessionNameFormat` arrives empty. Jasmine + * takes it from the worker's own service options, as legacy did. + */ + private sessionNameFormatFor(instance: TestFrameworkInstance, testContextOptions: TestContextOptions) { + if (testContextOptions.sessionNameFormat) { + return testContextOptions.sessionNameFormat + } + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + if (!frameworkName.toLowerCase().includes('jasmine')) { + return undefined + } + const format = (BrowserstackCLI.getInstance().options as { sessionNameFormat?: unknown })?.sessionNameFormat + return typeof format === 'function' ? format as TestContextOptions['sessionNameFormat'] : undefined + } + + private async annotate(browser: WebdriverIO.Browser, data: string) { + try { + await browser.executeScript(`browserstack_executor: ${JSON.stringify({ action: 'annotate', arguments: { data, level: 'info' } })}`, []) + } catch (error) { + this.logger.error(`annotate: failed to annotate the session: ${util.format(error)}`) + } + } + private isCucumberInstance(instance: TestFrameworkInstance): boolean { const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') return frameworkName.toLowerCase().includes('cucumber') @@ -341,13 +387,22 @@ export default class AutomateModule extends BaseModule { } } - async onAfterExecute() { + async onAfterExecute(args: { sessionVerdictInputs?: SessionVerdictInputs } = {}) { this.logger.debug('onAfterExecute: inside automate module after execute hook!') const userName = this.config.userName as string const accessKey = this.config.accessKey as string const testContextOptions = this.config.testContextOptions as TestContextOptions + if (args?.sessionVerdictInputs) { + // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options + const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } + const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true + await this.markJasmineSessions(this.liveSessionId(), this.jasmineVerdict(args.sessionVerdictInputs, ignoreHooksStatus)) + this.sessionMap.clear() + return + } + for (const [sessionId, sessionData] of this.sessionMap.entries()) { try { const failedTests = Array.from(sessionData.testResults.values()).filter(test => test.status === 'failed') @@ -389,6 +444,63 @@ export default class AutomateModule extends BaseModule { this.sessionMap.clear() } + private liveSessionId(): string { + const autoInstance = AutomationFramework.getTrackedInstance() + return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' + } + + /** + * Legacy `service.after()`'s session status: passed only when the runner reported no failed spec, a spec + * ran, and no test or hook failed, with `ignoreHooksStatus` handled as coded there. + */ + private jasmineVerdict(inputs: SessionVerdictInputs, ignoreHooksStatus: boolean): SessionVerdict { + const { result, specsRan, failReasons, pureTestFailReasons, hookFailReasons } = inputs + const joined = (reasons: string[]) => reasons.length > 0 ? reasons.join('\n') : undefined + + if (result === 0 && specsRan) { + const reasons = ignoreHooksStatus ? pureTestFailReasons : failReasons + return reasons.length > 0 ? { status: 'failed', reason: joined(reasons) } : { status: 'passed' } + } + if (ignoreHooksStatus && specsRan) { + const hasOnlyHookFailures = failReasons.length === 0 && hookFailReasons.length > 0 + if (hasOnlyHookFailures && pureTestFailReasons.length === 0) { + return { status: 'passed' } + } + return { status: 'failed', reason: joined(pureTestFailReasons) } + } + return { + status: 'failed', + reason: ignoreHooksStatus && pureTestFailReasons.length > 0 ? joined(pureTestFailReasons) : joined(failReasons), + } + } + + /** + * Jasmine: legacy `service.after()` marked only the live session, with the worker's status, the last + * name, and the test and hook failure reasons. A session nothing registered (a beforeAll failed before + * any spec ran) is still marked, and a reloaded one is left to the mark `onReload` already sent. + */ + private async markJasmineSessions(liveSessionId: string, verdict: SessionVerdict) { + const testContextOptions = this.config.testContextOptions as TestContextOptions + const auth = { user: this.config.userName as string, key: this.config.accessKey as string } + const sessionIds = new Set(this.sessionMap.keys()) + if (liveSessionId && isBrowserstackSession(AutomationFramework.getDriver(AutomationFramework.getTrackedInstance()) as WebdriverIO.Browser)) { + sessionIds.add(liveSessionId) + } + + for (const sessionId of sessionIds) { + try { + await this.flushSessionName(sessionId) + if (!liveSessionId || sessionId !== liveSessionId || testContextOptions.skipSessionStatus) { + continue + } + const name = testContextOptions.skipSessionName ? undefined : this.sessionMap.get(sessionId)?.lastTestName || undefined + await this.markSessionStatus(sessionId, verdict.status, verdict.reason, auth, name) + } catch (error) { + this.logger.error(`Failed to process session ${sessionId}: ${error}`) + } + } + } + // An App Automate session is identified by the service-level app / skipAppOverride flag, // OR an app supplied only via the appium:app / appium:options.app capability — which the // service-level config (this.config) does not carry. Mirrors accessibilityModule.isAppAutomateSession. @@ -474,10 +586,10 @@ export default class AutomateModule extends BaseModule { )(sessionId, sessionName, config) } - async markSessionStatus(sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }): Promise { + async markSessionStatus(sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }, sessionName?: string): Promise { return await PerformanceTester.measureWrapper( PERFORMANCE_SDK_EVENTS.AUTOMATE_EVENTS.SESSION_STATUS, - async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }) => { + async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }, sessionName?: string) => { try { const auth = Buffer.from(`${config.user}:${config.key}`).toString('base64') const { url: sessionStatusApiUrl, method, product } = this.resolveSessionApi(sessionId) @@ -485,6 +597,7 @@ export default class AutomateModule extends BaseModule { const body = { status: sessionStatus, + ...(sessionName ? { name: sessionName } : {}), ...(sessionErrorMessage ? { reason: sessionErrorMessage } : {}) } @@ -504,7 +617,7 @@ export default class AutomateModule extends BaseModule { this.logger.error(`Failed to update session status on BrowserStack: ${err}`) } } - )(sessionId, sessionStatus, sessionErrorMessage, config) + )(sessionId, sessionStatus, sessionErrorMessage, config, sessionName) } } diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index 5026bdaa..1830be1e 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -82,6 +82,10 @@ export default class TestHubModule extends BaseModule { } onBeforeTest(args: Record) { + // Jasmine reports to TestHub from its reporter feed; its module-only dispatches carry this marker + if (args.skipTestHub === true) { + return + } this.logger.debug('onBeforeTest: Called after test hook from cli configured module!!!') const autoInstance = AutomationFramework.getTrackedInstance() as AutomationFrameworkInstance const instances = [autoInstance] @@ -90,6 +94,9 @@ export default class TestHubModule extends BaseModule { } onAllTestEvents(args: Record) { + if (args.skipTestHub === true) { + return + } this.logger.debug('onAllTestEvents: Called after all test events from cli configured module!!!') const instance = args.instance as TestFrameworkInstance const testState = instance.getCurrentTestState() diff --git a/packages/browserstack-service/src/cli/skipReporter.ts b/packages/browserstack-service/src/cli/skipReporter.ts index 4fff79d5..7ec13504 100644 --- a/packages/browserstack-service/src/cli/skipReporter.ts +++ b/packages/browserstack-service/src/cli/skipReporter.ts @@ -69,7 +69,7 @@ async function emitSkipReport({ framework, test, result, suiteTitle }: QueuedSki // gated on it, not on TEST/POST) — same sequence afterTest uses const steps: Array<[State, State, Record]> = [ [TestFrameworkState.INIT_TEST, HookState.PRE, { test }], - [TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle }], + [TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle, skipReport: true }], [TestFrameworkState.LOG_REPORT, HookState.POST, { test, result }], [TestFrameworkState.TEST, HookState.POST, { test, result, suiteTitle }], ] diff --git a/packages/browserstack-service/src/constants.ts b/packages/browserstack-service/src/constants.ts index ae000252..1132ba04 100644 --- a/packages/browserstack-service/src/constants.ts +++ b/packages/browserstack-service/src/constants.ts @@ -42,7 +42,7 @@ export const DEFAULT_WAIT_TIMEOUT_FOR_PENDING_UPLOADS = 5000 // 5s export const DEFAULT_WAIT_INTERVAL_FOR_PENDING_UPLOADS = 100 // 100ms export const BSTACK_SERVICE_VERSION = bstackServiceVersion -export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName'] +export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName', 'sessionNamePrependTopLevelSuiteTitle', 'sessionNameOmitTestTitle'] export const BROWSERSTACK_TEST_PLAN_ID = 'BROWSERSTACK_TEST_PLAN_ID' export const LOGS_FILE = 'logs/bstack-wdio-service.log' diff --git a/packages/browserstack-service/src/insights-handler.ts b/packages/browserstack-service/src/insights-handler.ts index 3542882b..1dc36338 100644 --- a/packages/browserstack-service/src/insights-handler.ts +++ b/packages/browserstack-service/src/insights-handler.ts @@ -812,7 +812,8 @@ class _InsightsHandler { logEntry: { kind: TestFrameworkConstants.KIND_SCREENSHOT, message: result.value, - timestamp: new Date().toISOString() + timestamp: new Date().toISOString(), + ...(this._framework === 'jasmine' ? { test_run_uuid: testMeta.uuid } : {}) } }) : this.listener.onScreenshot([{ @@ -829,16 +830,29 @@ class _InsightsHandler { } // log http request + const httpResponse = { + path: requestData.endpoint, + method: requestData.method, + body, + response: result + } + // Only jasmine registers `command` on the CLI flow, and the legacy listener is inert there + if (BrowserstackCLI.getInstance().isRunning()) { + await BrowserstackCLI.getInstance().getTestFramework()?.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { + kind: 'HTTP', + message: JSON.stringify(httpResponse), + timestamp: new Date().toISOString(), + test_run_uuid: testMeta.uuid + } + }) + return + } this.listener.logCreated([{ test_run_uuid: testMeta.uuid, timestamp: new Date().toISOString(), kind: 'HTTP', - http_response: { - path: requestData.endpoint, - method: requestData.method, - body, - response: result - } + http_response: httpResponse }] ) } diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index 68d77738..1345c8b5 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -5,6 +5,10 @@ import WDIOReporter from '@wdio/reporter' import type { Options, Frameworks } from '@wdio/types' import { BrowserstackCLI } from './cli/index.js' import { reportSkippedTest, resolveSpecFile } from './cli/skipReporter.js' +import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' +import type { JasmineSuiteContext } from './cli/frameworks/wdioJasmineTestFramework.js' +import { TestFrameworkState } from './cli/states/testFrameworkState.js' +import { HookState } from './cli/states/hookState.js' import * as url from 'node:url' import { v4 as uuidv4 } from 'uuid' @@ -72,6 +76,10 @@ class _TestReporter extends WDIOReporter { } public async appendTestItemLog(stdLog: StdLog) { + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.LOG, HookState.POST, { source: 'reporter', logEntry: stdLog }) + return + } if (this._currentHook.uuid && !this._currentHook.finished) { stdLog.hook_run_uuid = this._currentHook.uuid } else if (_TestReporter.currentTest.uuid) { @@ -134,6 +142,29 @@ class _TestReporter extends WDIOReporter { this._suites.pop() } + /** + * WDIO waits for reporters to synchronise before the worker exits; the CLI jasmine feed dispatches + * asynchronously, so hold the worker until it has drained. Always true on every other path. + */ + get isSynchronised() { + return !this.isCliJasmine() || (this.cliJasmineFramework()?.isIdle() ?? true) + } + + /** On the CLI flow jasmine's test/hook/log events feed the framework tracker, never the legacy Listener. */ + isCliJasmine() { + return this._config?.framework === 'jasmine' && BrowserstackCLI.getInstance().isRunning() + } + + cliJasmineFramework() { + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioJasmineTestFramework ? framework : null + } + + jasmineSuiteContext(): JasmineSuiteContext { + const suiteFile = this._suiteName || (this.specs?.length > 0 ? this.specs[this.specs.length - 1]?.replace('file:', '') : undefined) + return { scopes: this._suites.map(s => s.title), suiteFile } + } + needToSendData(testType?: string, event?: string) { if (!this._observability) {return false} @@ -158,6 +189,10 @@ class _TestReporter extends WDIOReporter { } testStats.end ||= new Date() + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.POST, { source: 'reporter', testStats, context: this.jasmineSuiteContext() }) + return + } this.listener.testFinished(await this.getRunData(testStats, 'TestRunFinished')) } @@ -168,6 +203,17 @@ class _TestReporter extends WDIOReporter { if (testStats.fullTitle === '') { return } + if (this.isCliJasmine()) { + const args: Record = { source: 'reporter', testStats, context: this.jasmineSuiteContext() } + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + const cliUuid = args.testUuid as string | undefined + if (cliUuid) { + _TestReporter.currentTest.uuid = cliUuid + _TestReporter.currentTest.name = testStats.title + _TestReporter._tests[testStats.fullTitle] = { uuid: cliUuid } + } + return + } const uuid = uuidv4() _TestReporter.currentTest.uuid = uuid @@ -181,6 +227,10 @@ class _TestReporter extends WDIOReporter { if (!this.needToSendData('hook', 'start')) { return } + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), HookState.PRE, { source: 'reporter', hookStats, context: this.jasmineSuiteContext() }) + return + } const identifier = this.getHookIdentifier(hookStats) const hookId = uuidv4() @@ -196,6 +246,13 @@ class _TestReporter extends WDIOReporter { if (!this.needToSendData('hook', 'end')) { return } + if (this.isCliJasmine()) { + if (!hookStats.state && !hookStats.error) { + hookStats.state = 'passed' + } + this.cliJasmineFramework()?.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), HookState.POST, { source: 'reporter', hookStats, context: this.jasmineSuiteContext() }) + return + } const identifier = this.getHookIdentifier(hookStats) if (_TestReporter._tests[identifier]) { _TestReporter._tests[identifier].finishedAt = (new Date()).toISOString() diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index e262f5ca..16adc4bf 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -45,6 +45,7 @@ import { HookState } from './cli/states/hookState.js' import { AutomationFrameworkConstants } from './cli/frameworks/constants/automationFrameworkConstants.js' import TestFramework from './cli/frameworks/testFramework.js' import WdioCucumberTestFramework from './cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' import { TestFrameworkState } from './cli/states/testFrameworkState.js' import { TestFrameworkConstants } from './cli/frameworks/constants/testFrameworkConstants.js' import AccessibilityModule from './cli/modules/accessibilityModule.js' @@ -330,12 +331,22 @@ export default class BrowserstackService implements Services.ServiceInstance { * `browserCommand` is the only producer of TEST_SCREENSHOT logs — the binary * has no screenshot producer of its own — so the result event has to be * registered on this path too, or a screenshot taken mid-test never reaches - * Observability (SDK-4177). The `command` (beforeCommand) event is - * deliberately NOT registered: it only fills the map that browserCommand's - * HTTP-log half reads, and that half emits on the JS listener pipeline the - * binary owns here. Leaving it unregistered keeps the screenshot upload — - * which rides its own JWT-authenticated endpoint — as the single effect. + * Observability (SDK-4177). The `command` (beforeCommand) event only fills + * the map browserCommand's HTTP-log half reads, so it is registered for + * jasmine alone, whose HTTP command logs go over gRPC; mocha and cucumber + * never reported them on this path. */ + if (this._config.framework === 'jasmine') { + this._browser.on('command', (command) => { + if (shouldProcessEventForTesthub('')) { + this._insightsHandler?.browserCommand( + 'client:beforeCommand', + Object.assign(command, { sessionId }), + this._currentTest + ) + } + }) + } this._browser.on('result', (result) => { if (shouldProcessEventForTesthub('')) { this._insightsHandler?.browserCommand( @@ -450,7 +461,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeHook' }) - async beforeHook (test: Frameworks.Test|CucumberHook, context: unknown) { + async beforeHook (test: Frameworks.Test|CucumberHook, context: unknown, hookName?: string) { if (this._config.framework !== 'cucumber') { this._currentTest = test as Frameworks.Test // not update currentTest when this is called for cucumber step } @@ -481,7 +492,7 @@ export default class BrowserstackService implements Services.ServiceInstance { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) } } else if (framework) { - const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] + const hookFrameworkState = TestFrameworkState[this._cliHookType(test as Frameworks.Test, hookName) as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) } @@ -496,7 +507,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterHook' }) - async afterHook(test: Frameworks.Test | CucumberHook, context: unknown, result: Frameworks.TestResult) { + async afterHook(test: Frameworks.Test | CucumberHook, context: unknown, result: Frameworks.TestResult, hookName?: string) { // The Mocha hook window is closed — clear the tracker (see beforeHook). if (this._config.framework === 'mocha') { setCurrentMochaHookWindow(null) @@ -535,16 +546,16 @@ export default class BrowserstackService implements Services.ServiceInstance { return } if (framework) { - const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] + const hookType = this._cliHookType(test as Frameworks.Test, hookName) + const hookFrameworkState = TestFrameworkState[hookType as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result }) } // a failed (or skipping) before/each hook silently drops the suite's remaining // tests in mocha — report them as skipped so they surface on the dashboard and // attribute their Automate session (port of the legacy insights-handler cascade) - const hookType = getHookType((test as Frameworks.Test).title) const suite = (test as Frameworks.Test).ctx?.test?.parent - if (result && !result.passed && ['BEFORE_ALL', 'BEFORE_EACH', 'AFTER_EACH'].includes(hookType) && suite) { + if (this._config.framework === 'mocha' && result && !result.passed && ['BEFORE_ALL', 'BEFORE_EACH', 'AFTER_EACH'].includes(hookType) && suite) { await reportSuiteSkipped(framework, suite) } } @@ -555,6 +566,16 @@ export default class BrowserstackService implements Services.ServiceInstance { await this._accessibilityHandler?.afterHook() } + /** + * Jasmine's hook argument is a copy of the last started spec, never the hook itself, so only + * the `hookName` WDIO passes says which hook is running. + */ + private _cliHookType(test: Frameworks.Test, hookName?: string) { + return this._config.framework === 'jasmine' + ? WdioJasmineTestFramework.hookTypeFromName(hookName) + : getHookType(test.title) + } + /** * Cucumber abandons the whole feature when a `BeforeAll` throws, so every scenario in it * (Rule-nested included) is reported SKIPPED rather than vanishing. Ports @@ -770,7 +791,18 @@ export default class BrowserstackService implements Services.ServiceInstance { } catch (flushErr) { BStackLogger.debug(`Exception flushing deferred test finish in after(): ${util.format(flushErr)}`) } - await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.EXECUTE, HookState.POST, {}) + const executeArgs = this._config.framework === 'jasmine' + ? { + sessionVerdictInputs: { + result, + specsRan: this._specsRan, + failReasons: [...this._failReasons], + pureTestFailReasons: [...this._pureTestFailReasons], + hookFailReasons: [...this._hookFailReasons], + } + } + : {} + await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.EXECUTE, HookState.POST, executeArgs) } // if (setSessionStatus) { diff --git a/packages/browserstack-service/tests/cli/cliUtils.test.ts b/packages/browserstack-service/tests/cli/cliUtils.test.ts index aa82167c..99f23616 100644 --- a/packages/browserstack-service/tests/cli/cliUtils.test.ts +++ b/packages/browserstack-service/tests/cli/cliUtils.test.ts @@ -226,6 +226,56 @@ describe('CLIUtils', () => { testPlanId: 'tm-plan-456' }) }) + + describe('test plan id from env and CLI args', () => { + const CLI_ARG = '--browserstack.testManagementOptions.testPlanId' + const capabilities = [{ browserName: 'chrome' }] + let originalArgv: string[] + + beforeEach(() => { + originalArgv = process.argv + delete process.env.BROWSERSTACK_TEST_PLAN_ID + }) + + afterEach(() => { + process.argv = originalArgv + delete process.env.BROWSERSTACK_TEST_PLAN_ID + }) + + const planOf = (options = createBrowserstackOptions()) => + JSON.parse(CLIUtils.getBinConfig(mockConfig, capabilities, options)).testManagementOptions + + it('reads BROWSERSTACK_TEST_PLAN_ID', () => { + process.env.BROWSERSTACK_TEST_PLAN_ID = 'env-plan-1' + expect(planOf()).toEqual({ testPlanId: 'env-plan-1' }) + }) + + it('reads the CLI arg followed by its value', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', CLI_ARG, 'TP-123'] + expect(planOf()).toEqual({ testPlanId: 'TP-123' }) + }) + + it('reads the CLI arg in --key=value form', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', `${CLI_ARG}=12345`] + expect(planOf()).toEqual({ testPlanId: '12345' }) + }) + + it('prefers the CLI arg over the service option', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', CLI_ARG, 'cli-plan'] + expect(planOf(createBrowserstackOptions({ testManagementOptions: { testPlanId: 'option-plan' } }))) + .toEqual({ testPlanId: 'cli-plan' }) + }) + + it('omits testManagementOptions when no source sets an id', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js'] + expect(planOf()).toBeUndefined() + }) + + it('omits testManagementOptions when the env id is blank', () => { + process.env.BROWSERSTACK_TEST_PLAN_ID = ' ' + expect(planOf()).toBeUndefined() + }) + }) }) describe('getSdkVersion', () => { @@ -649,4 +699,14 @@ describe('CLIUtils', () => { expect(CLIUtils.getCliDir()).toBe('') }) }) + + describe('checkCLISupportedFrameworks', () => { + it.each(['mocha', 'cucumber', 'jasmine'])('routes %s to the CLI flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(true) + }) + + it.each([undefined, 'WebdriverIO-jasmine', 'Jasmine', 'unknown'])('keeps %s on the legacy flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(false) + }) + }) }) \ No newline at end of file diff --git a/packages/browserstack-service/tests/cli/grpcClient.test.ts b/packages/browserstack-service/tests/cli/grpcClient.test.ts index 94312c1c..0bf8b808 100644 --- a/packages/browserstack-service/tests/cli/grpcClient.test.ts +++ b/packages/browserstack-service/tests/cli/grpcClient.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import { GrpcClient } from '../../src/cli/grpcClient.js' +import { GrpcClient, stripTestPlanIdArg } from '../../src/cli/grpcClient.js' import { BStackLogger } from '../../src/cli/cliLogger.js' vi.mock('../../src/grpc/index.js', () => ({ @@ -241,3 +241,31 @@ describe('GrpcClient.logCreatedEvent', () => { }) }) }) + +describe('stripTestPlanIdArg', () => { + const ARG = '--browserstack.testManagementOptions.testPlanId' + + it('drops the arg and its value in the space form', () => { + expect(stripTestPlanIdArg(['run', 'wdio.conf.js', ARG, 'TP-123', '--spec', 'a.js'])) + .toEqual(['run', 'wdio.conf.js', '--spec', 'a.js']) + }) + + it('drops the arg in the = form', () => { + expect(stripTestPlanIdArg(['run', `${ARG}=TP-123`, 'wdio.conf.js'])) + .toEqual(['run', 'wdio.conf.js']) + }) + + it('leaves args untouched and in order when the arg is absent', () => { + const args = ['run', 'wdio.conf.js', '--browserstack.buildName', 'b1', '--logLevel=info'] + expect(stripTestPlanIdArg(args)).toEqual(args) + }) + + it('keeps a following flag when the arg has no value', () => { + expect(stripTestPlanIdArg(['run', ARG, '--spec', 'a.js'])) + .toEqual(['run', '--spec', 'a.js']) + }) + + it('handles the arg as the last token', () => { + expect(stripTestPlanIdArg(['run', 'wdio.conf.js', ARG])).toEqual(['run', 'wdio.conf.js']) + }) +}) diff --git a/packages/browserstack-service/tests/cli/index.test.ts b/packages/browserstack-service/tests/cli/index.test.ts index d4415b89..6c8bf92c 100644 --- a/packages/browserstack-service/tests/cli/index.test.ts +++ b/packages/browserstack-service/tests/cli/index.test.ts @@ -3,6 +3,12 @@ import * as bstackLogger from '../../src/bstackLogger.js' import { BStackLogger } from '../../src/cli/cliLogger.js' import { BrowserstackCLI } from '../../src/cli/index.js' +import { CLIUtils } from '../../src/cli/cliUtils.js' +import APIUtils from '../../src/cli/apiUtils.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' const bstackLoggerSpy = vi.spyOn(bstackLogger.BStackLogger, 'logToFile') bstackLoggerSpy.mockImplementation(() => {}) @@ -119,4 +125,73 @@ describe('BrowserstackCLI bootstrap error surfacing', () => { expect(instance.config.apis).toEqual({ automate: { api: 'https://flat-only.example' } }) }) }) + + describe('setupTestFramework', () => { + afterEach(() => { + instance.testFramework = null + }) + + const setup = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + instance.setupTestFramework() + return instance.getTestFramework() + } + + it('constructs the jasmine framework for WebdriverIO-jasmine', () => { + const framework = setup('WebdriverIO-jasmine') + expect(framework).toBeInstanceOf(WdioJasmineTestFramework) + expect(framework).not.toBeInstanceOf(WdioMochaTestFramework) + expect(framework).toBeInstanceOf(TestFramework) + expect(framework!.getTestFrameworks()).toEqual(['WebdriverIO-jasmine']) + expect(framework!.getTestFrameworksVersions()).toEqual({ 'WebdriverIO-jasmine': '9.0.0' }) + }) + + it('still constructs the mocha framework for WebdriverIO-mocha', () => { + expect(setup('WebdriverIO-mocha')).toBeInstanceOf(WdioMochaTestFramework) + }) + + it('still constructs the cucumber framework for WebdriverIO-cucumber', () => { + expect(setup('WebdriverIO-cucumber')).toBeInstanceOf(WdioCucumberTestFramework) + }) + + it('leaves an unknown name unregistered', () => { + expect(setup('WebdriverIO-unknown')).toBeNull() + }) + }) + + describe('loadModules TestHub wiring', () => { + const savedEnv = { ...process.env } + + afterEach(() => { + instance.testFramework = null + instance.modulesLoaded = false + process.env = { ...savedEnv } + }) + + const load = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + vi.spyOn(APIUtils, 'updateURLSForGRR').mockImplementation(() => {}) + vi.spyOn(CLIUtils, 'getAutomationFrameworkDetail').mockReturnValue({ name: 'WebdriverIO', version: { WebdriverIO: '9.0.0' } }) + instance.modulesLoaded = false + instance.loadModules({ + binSessionId: 'b1', + config: '{}', + testhub: { jwt: 'jwt', buildHashedId: 'build' } + } as any) + } + + it('hands the jasmine framework its TestHubModule once, when the module is built', () => { + const setTestHubModule = vi.spyOn(WdioJasmineTestFramework.prototype, 'setTestHubModule') + load('WebdriverIO-jasmine') + expect(setTestHubModule).toHaveBeenCalledTimes(1) + expect(setTestHubModule.mock.calls[0][0]).toBe(instance.modules.TestHubModule) + }) + + it('leaves the mocha framework untouched', () => { + const setTestHubModule = vi.spyOn(WdioJasmineTestFramework.prototype, 'setTestHubModule') + load('WebdriverIO-mocha') + expect(instance.modules.TestHubModule).toBeDefined() + expect(setTestHubModule).not.toHaveBeenCalled() + }) + }) }) diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index d4114756..5349c5e9 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -59,6 +59,7 @@ import AutomationFramework from '../../../src/cli/frameworks/automationFramework import { AutomationFrameworkState } from '../../../src/cli/states/automationFrameworkState.js' import { HookState } from '../../../src/cli/states/hookState.js' import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' describe('AccessibilityModule', () => { let accessibilityModule: AccessibilityModule @@ -194,6 +195,30 @@ describe('AccessibilityModule', () => { expect(accessibilityModule.accessibilityMap.get('session-w')).toBe(true) }) + it('opens no window for jasmine, whose classic flow scanned nothing before the first spec', async () => { + withA11yCaps() + const cliSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + getTestFramework: () => ({ getTestFrameworks: () => ['WebdriverIO-jasmine'] }) + } as any) + + await accessibilityModule.onBeforeExecute() + + expect(accessibilityModule.accessibilityMap.has('session-w')).toBe(false) + cliSpy.mockRestore() + }) + + it('still opens the window for mocha', async () => { + withA11yCaps() + const cliSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + getTestFramework: () => ({ getTestFrameworks: () => ['WebdriverIO-mocha'] }) + } as any) + + await accessibilityModule.onBeforeExecute() + + expect(accessibilityModule.accessibilityMap.get('session-w')).toBe(true) + cliSpy.mockRestore() + }) + it('respects autoScanning — the one validation the window still owns', async () => { withA11yCaps() accessibilityModule.autoScanning = false @@ -276,6 +301,25 @@ describe('AccessibilityModule', () => { }) }) + describe('onBeforeTest — jasmine spec name', () => { + it('filters on the spec description when the test has no title, and names the test with it', async () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockImplementation(() => 'session-w') + await accessibilityModule.onBeforeTest({ + suiteTitle: 'Nested outer', + test: { description: 'outer passing test', fullName: 'Nested outer outer passing test' } + }) + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Nested outer', 'outer passing test', {}, undefined, false) + }) + + it('keeps the title when there is one (mocha)', async () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockImplementation(() => 'session-w') + await accessibilityModule.onBeforeTest({ suiteTitle: 'Suite', test: { title: 't', description: 'ignored' } }) + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Suite', 't', {}, undefined, false) + }) + }) + describe('onBeforeExecute', () => { it('should patch browser methods when automation instance exists', async () => { vi.mocked(AutomationFramework.getState).mockImplementation((instance, key) => { diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index b0e27c76..aed20e4e 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -6,9 +6,11 @@ import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.j import { AutomationFrameworkState } from '../../../src/cli/states/automationFrameworkState.js' import { HookState } from '../../../src/cli/states/hookState.js' import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' +import { BStackLogger } from '../../../src/cli/cliLogger.js' import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' import type { Options } from '@wdio/types' // Mock dependencies @@ -21,6 +23,12 @@ vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ } })) +vi.mock('../../../src/cli/index.js', () => ({ + BrowserstackCLI: { + getInstance: vi.fn(() => ({ options: {} })) + } +})) + vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ default: { getTrackedInstance: vi.fn(), @@ -1005,3 +1013,243 @@ describe('AutomateModule preferScenarioName', () => { expect(namesPUT()).toContain('Can log in') }) }) + +describe('AutomateModule — jasmine session verdict', () => { + let automateModule: AutomateModule + const putBodies = () => vi.mocked(fetch).mock.calls.map(([url, opts]) => [String(url).split('/sessions/')[1], JSON.parse((opts as { body: string }).body)]) + const inputs = (overrides: Record = {}) => ({ + result: 0, specsRan: true, failReasons: [], pureTestFailReasons: [], hookFailReasons: [], ...overrides + }) + const verdict = (overrides: Record, ignoreHooksStatus: boolean) => + (automateModule as any).jasmineVerdict(inputs(overrides), ignoreHooksStatus) + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 'live' } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 'live' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'u', + accessKey: 'k' + } as any + }) + + describe('jasmineVerdict (legacy service.after())', () => { + it('passes a clean run with no reason, whatever ignoreHooksStatus says', () => { + expect(verdict({}, false)).toEqual({ status: 'passed' }) + expect(verdict({}, true)).toEqual({ status: 'passed' }) + }) + + it('fails a runner-clean run on any recorded reason, joined in order', () => { + expect(verdict({ failReasons: ['beforeAll hook failure', 'boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['beforeAll hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeAll hook failure\nboom' }) + }) + + it('under ignoreHooksStatus judges a runner-clean run on its pure test failures only', () => { + expect(verdict({ hookFailReasons: ['afterAll hook failure'] }, true)).toEqual({ status: 'passed' }) + expect(verdict({ failReasons: ['boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['afterAll hook failure'] }, true)) + .toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('under ignoreHooksStatus passes a runner-failed run whose only failures are hooks', () => { + expect(verdict({ result: 2, hookFailReasons: ['beforeEach hook failure'] }, true)).toEqual({ status: 'passed' }) + }) + + it('under ignoreHooksStatus fails a runner-failed run with pure failures, on those reasons', () => { + expect(verdict({ result: 1, failReasons: ['boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['hook'] }, true)) + .toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('under ignoreHooksStatus fails a runner-failed run with no recorded failure, with no reason', () => { + expect(verdict({ result: 1 }, true)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('fails a runner-failed run with every recorded reason when hooks count', () => { + expect(verdict({ result: 1, failReasons: ['beforeEach hook failure', 'boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['beforeEach hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeEach hook failure\nboom' }) + expect(verdict({ result: 1 }, false)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('fails a worker where no spec ran, preferring pure reasons under ignoreHooksStatus', () => { + expect(verdict({ specsRan: false }, false)).toEqual({ status: 'failed', reason: undefined }) + expect(verdict({ specsRan: false, failReasons: ['beforeAll hook failure'], hookFailReasons: ['beforeAll hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeAll hook failure' }) + expect(verdict({ specsRan: false, hookFailReasons: ['beforeAll hook failure'] }, true)).toEqual({ status: 'failed', reason: undefined }) + expect(verdict({ specsRan: false, failReasons: ['boom'], pureTestFailReasons: ['boom'] }, true)).toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('reproduces the CP0 hookfail verdict: failed, every hook failure in the order it happened', () => { + const hookFailures = ['beforeAll hook failure', 'beforeEach hook failure', 'beforeEach hook failure', 'afterEach hook failure', 'afterEach hook failure'] + expect(verdict({ result: 6, failReasons: hookFailures, hookFailReasons: hookFailures }, false)) + .toEqual({ status: 'failed', reason: hookFailures.join('\n') }) + }) + }) + + it('marks the live session with the verdict, the last name and the joined reasons', async () => { + (automateModule as any).sessionMap.set('live', { lastTestName: 'Hookfail afterEach suite', appliedName: 'Hookfail afterEach suite', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, failReasons: ['beforeAll hook failure', 'afterEach hook failure'] }) }) + + expect(putBodies()).toEqual([ + ['live.json', { status: 'failed', name: 'Hookfail afterEach suite', reason: 'beforeAll hook failure\nafterEach hook failure' }] + ]) + }) + + it('marks a live session no spec registered, without a name', async () => { + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, specsRan: false, failReasons: ['beforeAll hook failure'] }) }) + + expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'beforeAll hook failure' }]]) + }) + + it('leaves a reloaded session to its onReload mark, but still names it', async () => { + (automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs() }) + + expect(putBodies()).toEqual([ + ['old.json', { name: 'Suite A' }], + ['live.json', { status: 'passed' }], + ]) + }) + + it('marks no session status when no session is live', async () => { + vi.mocked(AutomationFramework.getState).mockReturnValue('' as any) + ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1 }) }) + + expect(putBodies()).toEqual([['old.json', { name: 'Suite A' }]]) + }) + + it('reads ignoreHooksStatus from the worker\'s service options', async () => { + vi.mocked(BrowserstackCLI.getInstance).mockReturnValueOnce({ options: { testObservabilityOptions: { ignoreHooksStatus: true } } } as any) + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, hookFailReasons: ['beforeEach hook failure'] }) }) + + expect(putBodies()).toEqual([['live.json', { status: 'passed' }]]) + }) + + it('honours skipSessionStatus and skipSessionName', async () => { + (automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', testResults: new Map(), scenariosRan: 0 }) + ;(automateModule.config as any).testContextOptions = { skipSessionName: true, skipSessionStatus: true } + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, failReasons: ['x'] }) }) + + expect(fetch).not.toHaveBeenCalled() + }) + + it('keeps the existing per-test aggregation when no verdict inputs arrive (mocha, cucumber)', async () => { + (automateModule as any).sessionMap.set('live', { + lastTestName: 'Suite', appliedName: 'Suite', scenariosRan: 0, + testResults: new Map([['t', { testName: 'Suite', status: 'failed', reason: 'boom' }]]) + }) + + await automateModule.onAfterExecute({}) + + expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'boom' }]]) + }) +}) + +describe('AutomateModule — per-test annotation', () => { + let automateModule: AutomateModule + let executeScript: ReturnType + const annotation = (data: string) => `browserstack_executor: ${JSON.stringify({ action: 'annotate', arguments: { data, level: 'info' } })}` + + const runBeforeTest = (frameworkName: string, test: Record, extra: Record = {}) => { + vi.mocked(TestFramework.getState).mockImplementation((_i, key) => key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME ? frameworkName : undefined) + return automateModule.onBeforeTest({ instance: {}, test, suiteTitle: 'Suite', ...extra }) + } + + beforeEach(() => { + vi.clearAllMocks() + executeScript = vi.fn().mockResolvedValue(undefined) + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 's1', executeScript } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 's1' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'u', + accessKey: 'k' + } as any + }) + + it('annotates a mocha test with its title, as legacy beforeTest did', async () => { + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(executeScript).toHaveBeenCalledExactlyOnceWith(annotation('Test: t'), []) + }) + + it('annotates a jasmine spec with its full name', async () => { + await runBeforeTest('WebdriverIO-jasmine', { description: 'outer passing test', fullName: 'Nested outer outer passing test' }) + expect(executeScript).toHaveBeenCalledExactlyOnceWith(annotation('Test: Nested outer outer passing test'), []) + }) + + it('annotates even when the session name is skipped', async () => { + (automateModule.config as any).testContextOptions.skipSessionName = true + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(executeScript).toHaveBeenCalledExactlyOnceWith(annotation('Test: t'), []) + }) + + it('never annotates cucumber, a skip report, or a non-BrowserStack session', async () => { + await runBeforeTest('WebdriverIO-cucumber', { title: 'Scenario', parent: 'Feature' }) + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }, { skipReport: true }) + vi.mocked(isBrowserstackSession).mockReturnValue(false) + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(executeScript).not.toHaveBeenCalled() + }) + + it('logs a failed annotate and still names the session', async () => { + executeScript.mockRejectedValue(new Error('annotate blew up')) + await expect(runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' })).resolves.toBeUndefined() + expect(BStackLogger.error).toHaveBeenCalledWith(expect.stringContaining('annotate blew up')) + expect((automateModule as any).sessionMap.get('s1')?.lastTestName).toBe('Suite - t') + }) +}) + +describe('AutomateModule — jasmine sessionNameFormat', () => { + let automateModule: AutomateModule + const format = vi.fn((_config: unknown, _caps: unknown, suiteTitle: string, testTitle?: string) => `fmt[${suiteTitle}][${String(testTitle)}]`) + const names = () => vi.mocked(fetch).mock.calls.map(([, opts]) => JSON.parse((opts as { body: string }).body).name) + + const runBeforeTest = (frameworkName: string, test: Record) => { + vi.mocked(TestFramework.getState).mockImplementation((_i, key) => key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME ? frameworkName : undefined) + return automateModule.onBeforeTest({ instance: {}, test, suiteTitle: 'Nested outer' }) + } + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 's1' } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 's1' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + vi.mocked(BrowserstackCLI.getInstance).mockReturnValue({ options: { sessionNameFormat: format } } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false, sessionNameFormat: '' }, + userName: 'u', + accessKey: 'k' + } as any + }) + + afterEach(() => { + vi.mocked(BrowserstackCLI.getInstance).mockReset().mockReturnValue({ options: {} } as any) + }) + + it('applies the service-option format for jasmine, with no test title (legacy call shape)', async () => { + await runBeforeTest('WebdriverIO-jasmine', { description: 'outer passing test', fullName: 'Nested outer outer passing test' }) + expect(names()).toEqual(['fmt[Nested outer][undefined]']) + }) + + it('leaves mocha on the binary config (format absent there)', async () => { + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(format).not.toHaveBeenCalled() + expect(names()).toEqual(['Suite - t']) + }) +}) diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts index 47316f20..c1031d39 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts @@ -152,6 +152,15 @@ describe('TestHubModule', () => { autoInstance: [mockAutomationInstance] }) }) + + it('should skip events marked skipTestHub', async () => { + const sendTestSessionEventSpy = vi.spyOn(testHubModule, 'sendTestSessionEvent').mockResolvedValue() + + await testHubModule.onBeforeTest({ test: { title: 'Test' } as Frameworks.Test, skipTestHub: true }) + + expect(AutomationFramework.getTrackedInstance).not.toHaveBeenCalled() + expect(sendTestSessionEventSpy).not.toHaveBeenCalled() + }) }) describe('onAllTestEvents', () => { @@ -225,6 +234,32 @@ describe('TestHubModule', () => { expect(sendTestFrameworkEventSpy).toHaveBeenCalledWith(mockArgs) }) + + it('should skip events marked skipTestHub', async () => { + const mockInstance = { + getCurrentTestState: vi.fn(() => TestFrameworkState.TEST), + getCurrentHookState: vi.fn(() => HookState.POST) + } + const sendTestFrameworkEventSpy = vi.spyOn(testHubModule, 'sendTestFrameworkEvent').mockResolvedValue() + + await testHubModule.onAllTestEvents({ instance: mockInstance, skipTestHub: true }) + + expect(mockInstance.getCurrentTestState).not.toHaveBeenCalled() + expect(sendTestFrameworkEventSpy).not.toHaveBeenCalled() + }) + + it('should only treat a literal true skipTestHub as the marker', async () => { + const mockInstance = { + getCurrentTestState: vi.fn(() => TestFrameworkState.TEST), + getCurrentHookState: vi.fn(() => HookState.PRE) + } + const sendTestFrameworkEventSpy = vi.spyOn(testHubModule, 'sendTestFrameworkEvent').mockResolvedValue() + const mockArgs = { instance: mockInstance, skipTestHub: 'true' } + + await testHubModule.onAllTestEvents(mockArgs) + + expect(sendTestFrameworkEventSpy).toHaveBeenCalledWith(mockArgs) + }) }) describe('sendTestFrameworkEvent', () => { diff --git a/packages/browserstack-service/tests/cli/skipReporter.test.ts b/packages/browserstack-service/tests/cli/skipReporter.test.ts index 6846e985..32a9f094 100644 --- a/packages/browserstack-service/tests/cli/skipReporter.test.ts +++ b/packages/browserstack-service/tests/cli/skipReporter.test.ts @@ -33,6 +33,7 @@ describe('skipReporter', () => { [TestFrameworkState.TEST, HookState.POST], ]) expect(calls[2][2]).toMatchObject({ result: { passed: false, skipped: true } }) + expect(calls.map(([, , args]) => (args as Record).skipReport)).toEqual([undefined, true, undefined, undefined]) }) it('does not re-report the same identifier', async () => { diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts new file mode 100644 index 00000000..25978192 --- /dev/null +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -0,0 +1,456 @@ +import path from 'node:path' +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +import * as bstackLogger from '../../src/bstackLogger.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../src/cli/states/hookState.js' +import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' +import TestHubModule from '../../src/cli/modules/testHubModule.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +const SUITE_FILE = path.join(process.cwd(), 'test/p2/nested.spec.js') +const BEFORE_ALL_REASON = 'Not run because a beforeAll function failed. The beforeAll failure will be reported on the suite that caused it.' + +interface Dispatch { + state: State + hook: State + data: Record + args: Record +} + +const snapshot = (instance: TestFrameworkInstance) => + JSON.parse(JSON.stringify(Object.fromEntries(instance.getAllData()), (_k, v) => v instanceof Map ? Object.fromEntries(v) : v)) + +const testStats = (overrides: Record = {}) => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + end: undefined as Date | undefined, + _duration: 0, + retries: 0, + state: 'pending', + ...overrides, +}) + +const hookStats = (title: string, overrides: Record = {}) => ({ + type: 'hook', + uid: title, + title, + start: new Date('2026-09-25T15:34:08.170Z'), + end: undefined as Date | undefined, + _duration: 0, + ...overrides, +}) + +const context = (scopes = ['Nested outer']) => ({ scopes, suiteFile: SUITE_FILE }) + +describe('WdioJasmineTestFramework', () => { + let framework: WdioJasmineTestFramework + // what reached TestHub (reporter path) and what reached the module observers (service path) + let dispatches: Dispatch[] + let moduleDispatches: Dispatch[] + let logSends: Array<{ state: string, entries: Record[], data: Record }> + let sessionEvents: number + let testHub: Record> + + const drain = () => framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, {}) + // The reporter does not await trackEvent: its events are applied before the call returns. + const reporterTestStart = (testStats: unknown, ctx: unknown) => { + const args: Record = { source: 'reporter', testStats, context: ctx } + framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + return args.testUuid as string | undefined + } + const reporterTestEnd = (testStats: unknown, ctx: unknown) => { + framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { source: 'reporter', testStats, context: ctx }) + } + const reporterHook = (hookState: State, hookStats: { title: string }, ctx: unknown) => { + framework.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), hookState, { source: 'reporter', hookStats, context: ctx }) + } + const reporterHookStart = (hookStats: { title: string }, ctx: unknown) => reporterHook(HookState.PRE, hookStats, ctx) + const reporterHookEnd = (hookStats: { title: string }, ctx: unknown = context()) => reporterHook(HookState.POST, hookStats, ctx) + const reporterLog = (logEntry: Record) => { + framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { source: 'reporter', logEntry }) + } + + beforeEach(() => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') + dispatches = [] + moduleDispatches = [] + logSends = [] + sessionEvents = 0 + testHub = { + onBeforeTest: vi.fn(() => { sessionEvents++ }), + sendTestFrameworkEvent: vi.fn(async (args: Record) => { + const instance = args.instance as TestFrameworkInstance + dispatches.push({ state: instance.getCurrentTestState(), hook: instance.getCurrentHookState(), data: snapshot(instance), args }) + return true + }), + sendLogCreatedEvent: vi.fn(async (args: Record) => { + const instance = args.instance as TestFrameworkInstance + logSends.push({ state: instance.getCurrentTestState().toString(), entries: args.logEntries as Record[], data: snapshot(instance) }) + }), + } + framework.setTestHubModule(testHub as unknown as TestHubModule) + vi.spyOn(framework, 'runHooks').mockImplementation(async (instance, state, hook, args) => { + moduleDispatches.push({ state, hook, data: snapshot(instance), args: args as Record }) + }) + }) + + afterEach(() => { + vi.restoreAllMocks() + delete process.env.BROWSERSTACK_OBSERVABILITY + }) + + it('sends legacy identity for a spec, char-for-char with the CP0 TestRun', async () => { + reporterTestStart(testStats(), context()) + await drain() + + expect(dispatches).toHaveLength(1) + const [{ state, hook, data, args }] = dispatches + expect([state, hook]).toEqual([TestFrameworkState.TEST, HookState.PRE]) + expect(data.test_name).toBe('outer passing test') + expect(data.test_id).toBe('Nested outer outer passing test') + expect(data.test_scope).toBe('Nested outer outer passing test') + expect(data.test_scopes).toEqual(['Nested outer']) + expect(data.test_tags).toEqual([]) + expect(data.test_file_path).toBe(SUITE_FILE) + expect(data.test_location).toBe('test/p2/nested.spec.js') + expect(data.test_framework_name).toBe('WebdriverIO-jasmine') + expect(data.test_retries).toEqual({ limit: 0, attempts: 0 }) + expect(data.test_started_at).toBe('2026-09-25T15:34:12.259Z') + expect(data.test_duration).toBe(0) + expect(data.test_result).toBe('pending') + expect('test_code' in data).toBe(false) + expect(args.suiteTitle).toBe('Nested outer') + expect(args.test).toEqual({ description: 'outer passing test', fullName: 'Nested outer outer passing test', file: SUITE_FILE }) + }) + + it('keeps @tags from describes and the spec, sigil included', async () => { + reporterTestStart(testStats({ title: 'tagged @smoke', fullTitle: 'Outer @regression tagged @smoke' }), context(['Outer @regression'])) + await drain() + expect(dispatches[0].data.test_tags).toEqual(['@regression', '@smoke']) + }) + + it('finishes a passed spec on the same uuid with reporter timing and the result timestamp', async () => { + const stats = testStats() + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date('2026-09-25T15:34:16.645Z'), _duration: 4386 }, context()) + await drain() + + expect(dispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) + const [start, finish] = dispatches + expect(finish.data.test_uuid).toBe(start.data.test_uuid) + expect(finish.data.test_result).toBe('passed') + expect(finish.data.test_started_at).toBe('2026-09-25T15:34:12.259Z') + expect(finish.data.test_ended_at).toBe('2026-09-25T15:34:16.645Z') + expect(finish.data.test_duration).toBe(4386) + expect(typeof finish.data.test_result_at).toBe('string') + expect('test_failure' in finish.data).toBe(false) + expect(finish.args.result).toMatchObject({ passed: true, skipped: false }) + }) + + it('reports a pending spec as Start + Finish(skipped)', async () => { + reporterTestStart(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test' }), context(['Pending suite'])) + // @wdio/reporter replaces the TestStats object on test:pending; only the uid carries over + reporterTestEnd(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test', state: 'skipped', end: new Date() }), context(['Pending suite'])) + await drain() + + expect(dispatches).toHaveLength(2) + expect(dispatches[1].data.test_result).toBe('skipped') + expect(dispatches[1].data.test_duration).toBe(0) + expect(dispatches[1].args.result).toMatchObject({ passed: false, skipped: true }) + }) + + it('keeps jasmine\'s exact reason on a beforeAll-failed child', async () => { + const error = { message: BEFORE_ALL_REASON, stack: '' } + reporterTestStart(testStats({ uid: 'child0' }), context()) + reporterTestEnd(testStats({ uid: 'child0', state: 'failed', error, end: new Date() }), context()) + await drain() + + const finish = dispatches[1].data + expect(finish.test_result).toBe('failed') + expect(finish.test_failure_reason).toBe(BEFORE_ALL_REASON) + expect(finish.test_failure_type).toBe('UnhandledError') + expect(finish.test_failure).toEqual([{ backtrace: [BEFORE_ALL_REASON, ''] }]) + }) + + it('maps an AssertionError message to failure_type AssertionError', async () => { + reporterTestStart(testStats(), context()) + reporterTestEnd(testStats({ state: 'failed', error: { message: 'AssertionError: nope', stack: 'at x' } }), context()) + await drain() + expect(dispatches[1].data.test_failure_type).toBe('AssertionError') + }) + + it('gives overlapping specs distinct instances and closes each on its own uuid', async () => { + reporterTestStart(testStats({ uid: 'a' }), context()) + reporterTestStart(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b' }), context()) + reporterTestEnd(testStats({ uid: 'a', state: 'passed', end: new Date() }), context()) + reporterTestEnd(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b', state: 'passed', end: new Date() }), context()) + await drain() + + const uuidOf = (i: number) => dispatches[i].data.test_uuid + expect(uuidOf(0)).not.toBe(uuidOf(1)) + expect(uuidOf(2)).toBe(uuidOf(0)) + expect(uuidOf(3)).toBe(uuidOf(1)) + expect(dispatches[2].data.test_id).toBe('Nested outer outer passing test') + }) + + it('reports beforeAll/afterAll with legacy hook identity and no test linkage', async () => { + const before = hookStats('"before all" hook') + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'passed', end: new Date('2026-09-25T15:34:12.257Z'), _duration: 4087 }) + const after = hookStats('"after all" hook') + reporterHookStart(after, context(['Nested outer', 'Nested middle'])) + reporterHookEnd({ ...after, state: 'passed', end: new Date() }) + await drain() + + expect(dispatches.map(d => [d.state, d.hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.PRE], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + const started = (dispatches[0].data.test_hooks_started as Record[]>).BEFORE_ALL[0] + expect(started).toMatchObject({ + key: 'BEFORE_ALL', + hook_name: '"before all" hook', + hook_identifier: '"before all" hook for Nested outer', + hook_scope: 'Nested outer - "before all" hook', + hook_scopes: ['Nested outer'], + hook_result: 'pending', + event_started_at: '2026-09-25T15:34:08.170Z', + test_file_path: SUITE_FILE, + }) + const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] + expect(finished).toMatchObject({ hook_result: 'passed', event_ended_at: '2026-09-25T15:34:12.257Z', hook_duration: 4087 }) + expect(finished.hook_id).toBe(started.hook_id) + expect('hook_failure' in finished).toBe(false) + // the session event is keyed by the hook's own uuid, so the hook run gets its platform and session (#23); + // the binary builds the HookRun from hook_id and never links it to a test (#25) + expect(dispatches[0].data.test_uuid).toBe(started.hook_id) + expect(testHub.onBeforeTest).toHaveBeenCalledTimes(2) + expect(testHub.onBeforeTest.mock.calls.map(([args]) => (args as Record).instance)).toEqual([dispatches[0].args.instance, dispatches[2].args.instance]) + + const afterStarted = (dispatches[2].data.test_hooks_started as Record[]>).AFTER_ALL[0] + expect(afterStarted.hook_identifier).toBe('"after all" hook for Nested middle') + expect(afterStarted.hook_scope).toBe('Nested outer - "after all" hook') + }) + + it('carries failure fields on a failed beforeAll', async () => { + const before = hookStats('"before all" hook') + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'failed', error: { message: 'boom', stack: 'Error: boom' }, end: new Date() }) + await drain() + const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] + expect(finished).toMatchObject({ + hook_result: 'failed', + hook_failure: [{ backtrace: ['boom', 'Error: boom'] }], + hook_failure_reason: 'boom', + hook_failure_type: 'UnhandledError', + }) + }) + + it('never reports each-hooks, or a hook outside any describe', async () => { + reporterHookStart(hookStats('"before each" hook'), context()) + reporterHookStart(hookStats('"before all" hook'), context([])) + await drain() + expect(dispatches).toHaveLength(0) + }) + + it('suppresses hook and log families in an accessibility-only run, like the legacy Listener', async () => { + delete process.env.BROWSERSTACK_OBSERVABILITY + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + try { + const before = hookStats('"before all" hook') + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'passed', end: new Date() }) + reporterTestStart(testStats(), context()) + reporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) + await drain() + expect(dispatches.map(d => d.state)).toEqual([TestFrameworkState.TEST]) + expect(logSends).toHaveLength(0) + } finally { + delete process.env.BROWSERSTACK_ACCESSIBILITY + } + }) + + it('sends reporter events to TestHub only, never through the module observers', async () => { + const stats = testStats() + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date() }, context()) + await drain() + expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(2) + expect(sessionEvents).toBe(1) + expect(moduleDispatches).toHaveLength(0) + }) + + it('drives the modules from the service hooks on the reporter\'s instance, with TestHub skipped', async () => { + const spec = { description: 'outer passing test', fullName: 'Nested outer outer passing test' } + reporterTestStart(testStats(), context()) + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) + const uuid = dispatches[0].data.test_uuid + expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) + expect(process.env.TEST_ANALYTICS_ID).toBe(uuid) + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: spec, suiteTitle: 'Nested outer' }) + await framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test: spec, result: {} }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: spec, result: { passed: true }, suiteTitle: 'Jasmine__TopLevel__Suite' }) + + expect(moduleDispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) + for (const d of moduleDispatches) { + expect(d.args.skipTestHub).toBe(true) + expect(d.data.test_uuid).toBe(uuid) + } + expect(moduleDispatches[1].args.suiteTitle).toBe('Nested outer') + expect(moduleDispatches[1].args.result).toEqual({ passed: true }) + expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(1) + }) + + it('never shows a spec the reporter saw but the service did not (pending, excluded, beforeAll-failed) to the modules', async () => { + reporterTestStart(testStats({ uid: 'x' }), context()) + reporterTestEnd(testStats({ uid: 'x', state: 'skipped', end: new Date() }), context()) + await drain() + expect(dispatches).toHaveLength(2) + expect(moduleDispatches).toHaveLength(0) + }) + + it('still drives the modules when Test Observability is opted out and the reporter feeds nothing', async () => { + const spec = { description: 'a', fullName: 'Suite a', file: SUITE_FILE } + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: spec, suiteTitle: 'Suite' }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: spec, result: { passed: true } }) + expect(moduleDispatches).toHaveLength(2) + expect(moduleDispatches[0].data.test_uuid).toBe(moduleDispatches[1].data.test_uuid) + expect(process.env.TEST_ANALYTICS_ID).toBe(moduleDispatches[0].data.test_uuid) + expect(testHub.sendTestFrameworkEvent).not.toHaveBeenCalled() + }) + + it('sends an open all-hook\'s logs in the hook state, else the last-started spec\'s in the test state', async () => { + reporterTestStart(testStats(), context()) + reporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) + const after = hookStats('"after all" hook') + reporterHookStart(after, context()) + reporterLog({ level: 'INFO', message: 'in hook', timestamp: 't2', kind: 'TEST_LOG' }) + reporterHookEnd({ ...after, state: 'passed', end: new Date() }) + reporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) + await drain() + + expect(logSends.map(l => l.state)).toEqual(['TestFrameworkState.TEST', 'TestFrameworkState.AFTER_ALL', 'TestFrameworkState.TEST']) + const specUuid = dispatches[0].data.test_uuid + expect(logSends[0].data.test_uuid).toBe(specUuid) + expect(logSends[0].entries[0]).not.toHaveProperty('hook_id') + const hookId = (dispatches.find(d => d.state === TestFrameworkState.AFTER_ALL)!.data.test_hooks_started as Record[]>).AFTER_ALL[0].hook_id + expect(logSends[1].entries[0].hook_id).toBe(hookId) + expect(logSends[2].data.test_uuid).toBe(specUuid) + }) + + it('keeps a screenshot entry\'s kind on the log path', async () => { + reporterTestStart(testStats(), context()) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) + await drain() + expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') + }) + + it('sends an HTTP command log to the spec it names, even inside an all-hook or after the spec ended', async () => { + const first = testStats() + const uuid = reporterTestStart(first, context()) + reporterTestEnd({ ...first, state: 'passed', end: new Date() }, context()) + const after = hookStats('"after all" hook') + reporterHookStart(after, context()) + const message = JSON.stringify({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message, timestamp: 't', test_run_uuid: uuid } }) + await drain() + + expect(logSends).toHaveLength(1) + expect(logSends[0].state).toBe('TestFrameworkState.TEST') + expect(logSends[0].data.test_uuid).toBe(uuid) + expect(logSends[0].entries[0]).toMatchObject({ kind: 'HTTP', timestamp: 't' }) + expect(logSends[0].entries[0]).not.toHaveProperty('hook_id') + expect(Buffer.from(logSends[0].entries[0].message as Uint8Array).toString()).toBe(message) + }) + + it('drops a named-spec log whose uuid this worker never minted', async () => { + reporterTestStart(testStats(), context()) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message: '{}', timestamp: 't', test_run_uuid: 'not-ours' } }) + await drain() + expect(logSends).toHaveLength(0) + }) + + it('maps the WDIO hookName to the hook-type key, and nothing else', () => { + expect(['beforeAll', 'afterAll', 'beforeEach', 'afterEach'].map(n => WdioJasmineTestFramework.hookTypeFromName(n))) + .toEqual(['BEFORE_ALL', 'AFTER_ALL', 'BEFORE_EACH', 'AFTER_EACH']) + expect(WdioJasmineTestFramework.hookTypeFromName(undefined)).toBe('unknown') + expect(WdioJasmineTestFramework.hookTypeFromName('constructor')).toBe('unknown') + }) + + it('ignores the service\'s hook events: hooks reach TestHub from the reporter only', async () => { + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { fullName: 'Nested outer outer passing test' } }) + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { test: {}, result: { passed: false } }) + expect(dispatches).toHaveLength(0) + expect(moduleDispatches).toHaveLength(0) + }) + + it('is not idle until queued events are dispatched', async () => { + reporterTestStart(testStats(), context()) + expect(framework.isIdle()).toBe(false) + await drain() + expect(framework.isIdle()).toBe(true) + }) + + it('counts pending events per framework instance', async () => { + const other = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') + reporterTestStart(testStats(), context()) + expect(framework.isIdle()).toBe(false) + expect(other.isIdle()).toBe(true) + await drain() + }) + + it('mints a reporter spec before trackEvent returns, so the uuid and the service lookup are immediate', async () => { + const args: Record = { source: 'reporter', testStats: testStats(), context: context() } + const pending = framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + const uuid = args.testUuid + expect(uuid).toEqual(expect.any(String)) + expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) + expect(dispatches).toHaveLength(0) + + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: { description: 'outer passing test', fullName: 'Nested outer outer passing test' } }) + await pending + expect(process.env.TEST_ANALYTICS_ID).toBe(uuid) + expect(dispatches[0].data.test_uuid).toBe(uuid) + }) + + it('leaves no uuid on the args when minting fails, and logs instead of throwing', async () => { + const args: Record = { source: 'reporter', testStats: undefined, context: context() } + await expect(framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args)).resolves.toBeUndefined() + expect(args.testUuid).toBeUndefined() + expect(framework.isIdle()).toBe(true) + }) + + it('reports a hook in BEFORE_ALL/AFTER_ALL and every other hook in NONE', () => { + expect(['"before all" hook', '"after all" hook', '"before each" hook', '"after each" hook', undefined].map(t => WdioJasmineTestFramework.reporterHookState(t))) + .toEqual([TestFrameworkState.BEFORE_ALL, TestFrameworkState.AFTER_ALL, TestFrameworkState.NONE, TestFrameworkState.NONE, TestFrameworkState.NONE]) + }) + + it('logs and continues when an observer throws', async () => { + testHub.sendTestFrameworkEvent.mockRejectedValueOnce(new Error('send blew up')) + const stats = testStats() + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date() }, context()) + await drain() + expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) + expect(framework.isIdle()).toBe(true) + }) + + // The class calls these TestHubModule methods directly; renaming or removing one must fail here. + it('relies on TestHubModule handlers that exist', () => { + for (const method of ['onBeforeTest', 'sendTestFrameworkEvent', 'sendLogCreatedEvent']) { + expect(typeof (TestHubModule.prototype as unknown as Record)[method]).toBe('function') + } + }) +}) diff --git a/packages/browserstack-service/tests/insights-handler.test.ts b/packages/browserstack-service/tests/insights-handler.test.ts index 7034c976..6d081093 100644 --- a/packages/browserstack-service/tests/insights-handler.test.ts +++ b/packages/browserstack-service/tests/insights-handler.test.ts @@ -10,6 +10,8 @@ import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' import { BrowserstackCLI } from '../src/cli/index.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' const log = logger('test') let insightsHandler: InsightsHandler @@ -736,6 +738,101 @@ describe('browserCommand', () => { insightsHandler.browserCommand('client:afterCommand', { sessionId: 's', method: 'm', endpoint: 'e', result: { value: 'random' } }, {}) expect(uploadEventDataSpy).toBeCalledTimes(0) }) + + describe('on the CLI flow', () => { + const cliBrowser = { on: vi.fn(), sessionId: 's', capabilities: {}, config: {}, execute: vi.fn() } as any + const command = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/title', body: {} } + const result = { ...command, result: { value: 'StackDemo' } } + const lastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any + let getInstanceSpy: ReturnType | undefined + + const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + } + + const handlerFor = (framework: string) => { + const handler = new InsightsHandler(cliBrowser, framework) + handler['getIdentifier'] = vi.fn().mockReturnValue('Nested outer outer passing test') + handler['_tests'] = { 'Nested outer outer passing test': { uuid: 'spec-uuid' } } + return handler + } + + beforeEach(() => { + delete process.env[TESTOPS_SCREENSHOT_ENV] + commandSpy.mockRestore() + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).not.toHaveBeenCalled() + expect(trackEvent).toHaveBeenCalledTimes(1) + const [state, hook, { logEntry }] = trackEvent.mock.calls[0] + expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) + expect(logEntry.kind).toBe('HTTP') + expect(logEntry.test_run_uuid).toBe('spec-uuid') + expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + }) + + it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).toHaveBeenCalledWith([{ + test_run_uuid: 'spec-uuid', + timestamp: expect.any(String), + kind: 'HTTP', + http_response: { path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } } + }]) + }) + + it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { + process.env[TESTOPS_SCREENSHOT_ENV] = 'true' + const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } + + const jasmineTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: jasmineTrack }) + await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) + getInstanceSpy!.mockRestore() + + const mochaTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: mochaTrack }) + await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) + + expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) + expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) + }) + + it('drops commands with no spec yet, as legacy did before the first spec', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + handler['getIdentifier'] = vi.fn().mockReturnValue(undefined) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, {} as any) + await handler.browserCommand('client:afterCommand', { ...result } as any, {} as any) + + expect(trackEvent).not.toHaveBeenCalled() + }) + }) }) describe('getIdentifier', () => { diff --git a/packages/browserstack-service/tests/reporter.test.ts b/packages/browserstack-service/tests/reporter.test.ts index da59c2c2..5f8a169a 100644 --- a/packages/browserstack-service/tests/reporter.test.ts +++ b/packages/browserstack-service/tests/reporter.test.ts @@ -4,6 +4,10 @@ import { describe, expect, it, vi, beforeEach, afterEach, beforeAll, afterAll } import type { StdLog } from '../src/index.js' import TestReporter from '../src/reporter.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' @@ -354,4 +358,222 @@ describe('test-reporter', () => { expect(sendDataSpy).toBeCalledTimes(0) }) }) + + describe('jasmine CLI feed', () => { + const jasmineRunnerConfig = { + type: 'runner', + cid: '0-0', + capabilities: { browserName: 'chrome', browserVersion: '151' }, + config: { framework: 'jasmine', hostname: 'hub.browserstack.com' }, + specs: ['/work/test/p2/nested.spec.js'], + sessionId: 'sessionId' + } + const suite = { title: 'Nested outer', file: '/work/test/p2/nested.spec.js' } + const jasmineTestStats = () => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + _duration: 0, + retries: 0, + state: 'pending', + }) + const hookStats = () => ({ type: 'hook', uid: 'h', title: '"before all" hook', start: new Date(), _duration: 0 }) + const logEntry = () => ({ timestamp: new Date().toISOString(), level: 'INFO', message: 'console line', kind: 'TEST_LOG' as const, http_response: {} }) + + let reporter: TestReporter + let listener: Record> + let framework: WdioJasmineTestFramework + let getInstanceSpy: ReturnType | undefined + let getGitMetaDataSpy: ReturnType + // the reporter-sourced trackEvent calls, as [state, hookState, args] + const reporterCalls = () => vi.mocked(framework.trackEvent).mock.calls.filter(([, , args]) => (args as Record)?.source === 'reporter') + + const setCli = (running: boolean, testFramework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => running, + getTestFramework: () => testFramework, + } as unknown as BrowserstackCLI) + } + + beforeEach(async () => { + vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') + getGitMetaDataSpy = vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') + vi.spyOn(framework, 'trackEvent').mockImplementation(async (state, hookState, args = {}) => { + if (state === TestFrameworkState.TEST && hookState === HookState.PRE) { + args.testUuid = 'cli-uuid' + } + }) + + reporter = new TestReporter({}) + await reporter.onRunnerStart(jasmineRunnerConfig as any) + reporter.onSuiteStart(suite as any) + listener = { + testStarted: vi.spyOn(reporter['listener'], 'testStarted').mockImplementation(() => {}), + testFinished: vi.spyOn(reporter['listener'], 'testFinished').mockImplementation(() => {}), + hookStarted: vi.spyOn(reporter['listener'], 'hookStarted').mockImplementation(() => {}), + hookFinished: vi.spyOn(reporter['listener'], 'hookFinished').mockImplementation(() => {}), + logCreated: vi.spyOn(reporter['listener'], 'logCreated').mockImplementation(() => {}), + } + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + getGitMetaDataSpy.mockRestore() + for (const spy of Object.values(listener)) { + spy.mockRestore() + } + }) + + describe('on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('feeds every test and hook event to the framework and enqueues nothing on the legacy Listener', async () => { + const stats = jasmineTestStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + await reporter.appendTestItemLog(logEntry()) + + expect(reporterCalls().map(([state, hookState]) => [state, hookState])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + [TestFrameworkState.TEST, HookState.PRE], + [TestFrameworkState.TEST, HookState.POST], + [TestFrameworkState.LOG, HookState.POST], + ]) + expect(framework.trackEvent).toHaveBeenCalledTimes(5) + for (const spy of Object.values(listener)) { + expect(spy).not.toHaveBeenCalled() + } + }) + + it('passes the suite stack and suite file', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect(framework.trackEvent).toHaveBeenCalledWith(TestFrameworkState.TEST, HookState.PRE, expect.objectContaining({ + source: 'reporter', + testStats: expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), + context: { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' }, + })) + }) + + it('passes hook stats with the suite context, and log entries as they came', async () => { + const hook = hookStats() + await reporter.onHookStart(hook as any) + await reporter.onHookEnd(hook as any) + const entry = logEntry() + await reporter.appendTestItemLog(entry) + const context = { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } + expect(reporterCalls().map(([, , args]) => args)).toEqual([ + { source: 'reporter', hookStats: hook, context }, + { source: 'reporter', hookStats: hook, context }, + { source: 'reporter', logEntry: entry }, + ]) + }) + + it('reports an each-hook in NONE, leaving the class to drop it', async () => { + await reporter.onHookStart({ ...hookStats(), title: '"before each" hook' } as any) + expect(framework.trackEvent).toHaveBeenCalledWith(TestFrameworkState.NONE, HookState.PRE, expect.objectContaining({ source: 'reporter' })) + }) + + it('records nothing for the spec when the framework minted no uuid', async () => { + vi.mocked(framework.trackEvent).mockResolvedValue(undefined) + ;(TestReporter as any).currentTest = {} + await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: 'No uuid spec' } as any) + expect(TestReporter.getTests()['No uuid spec']).toBeUndefined() + expect((TestReporter as any).currentTest).toEqual({}) + }) + + it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) + }) + + it('names the current test for Percy\'s testCase, as legacy getRunData did', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect((TestReporter as any).currentTest).toMatchObject({ uuid: 'cli-uuid', name: 'outer passing test' }) + }) + + it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { + const stats = jasmineTestStats() as Record + await reporter.onTestEnd(stats as any) + expect(stats.end).toBeInstanceOf(Date) + const hook = hookStats() as Record + await reporter.onHookEnd(hook as any) + expect(hook.state).toBe('passed') + }) + + it('drops ', async () => { + await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: '' } as any) + await reporter.onTestEnd({ ...jasmineTestStats(), fullTitle: '' } as any) + expect(framework.trackEvent).not.toHaveBeenCalled() + }) + + it('sends nothing when Test Observability is opted out', async () => { + reporter['_observability'] = false + await reporter.onTestStart(jasmineTestStats() as any) + expect(framework.trackEvent).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + }) + + it('never falls back to the legacy Listener when the tracker is missing', async () => { + setCli(true, null) + await reporter.onTestStart(jasmineTestStats() as any) + await reporter.appendTestItemLog(logEntry()) + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.logCreated).not.toHaveBeenCalled() + }) + + it('reports unsynchronised while the framework has queued events', () => { + const isIdle = vi.spyOn(framework, 'isIdle').mockReturnValue(false) + expect(reporter.isSynchronised).toBe(false) + isIdle.mockReturnValue(true) + expect(reporter.isSynchronised).toBe(true) + }) + + it('is synchronised when the CLI test framework is not jasmine\'s', () => { + setCli(true, { isIdle: () => false }) + expect(reporter.isSynchronised).toBe(true) + }) + }) + + describe('on the legacy flow', () => { + beforeEach(() => setCli(false, null)) + + it('enqueues on the legacy Listener and never touches the framework', async () => { + const stats = jasmineTestStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + reporter['_currentHook'] = {} + await reporter.appendTestItemLog(logEntry()) + + expect(listener.hookStarted).toHaveBeenCalledTimes(1) + expect(listener.hookFinished).toHaveBeenCalledTimes(1) + expect(listener.testStarted).toHaveBeenCalledTimes(1) + expect(listener.testFinished).toHaveBeenCalledTimes(1) + expect(listener.logCreated).toHaveBeenCalledTimes(1) + expect(framework.trackEvent).not.toHaveBeenCalled() + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: '123456789' }) + }) + }) + + describe('mocha on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('leaves the mocha arm unchanged: no start/end publishing from the reporter', async () => { + reporter['_config']!.framework = 'mocha' + await reporter.onTestStart(jasmineTestStats() as any) + await reporter.onTestEnd({ ...jasmineTestStats(), state: 'passed' } as any) + expect(framework.trackEvent).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.testFinished).not.toHaveBeenCalled() + }) + }) + }) }) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index aa32efbc..1a146ab8 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -7,6 +7,8 @@ import BrowserstackService from '../src/service.js' import * as utils from '../src/util.js' import InsightsHandler from '../src/insights-handler.js' import { BrowserstackCLI } from '../src/cli/index.js' +import * as skipReporter from '../src/cli/skipReporter.js' +import TestFramework from '../src/cli/frameworks/testFramework.js' import AccessibilityModule from '../src/cli/modules/accessibilityModule.js' import * as bstackLogger from '../src/bstackLogger.js' import AutomationFramework from '../src/cli/frameworks/automationFramework.js' @@ -15,11 +17,14 @@ import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' import { HookState } from '../src/cli/states/hookState.js' import { AutomationFrameworkConstants } from '../src/cli/frameworks/constants/automationFrameworkConstants.js' import { AutomationFrameworkState } from '../src/cli/states/automationFrameworkState.js' +import PerformanceTester from '../src/instrumentation/performance/performance-tester.js' const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' const sessionBaseUrl = 'https://api.browserstack.com/automate/sessions' const sessionId = 'session123' const sessionIdA = 'session456' +// What WDIO hands a jasmine hook: a copy of the last started spec (no `title`), then the hookName. +const jasmineLastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any vi.mock('fetch') vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) @@ -850,6 +855,41 @@ describe('before', () => { expect(browserA.overwriteCommand).not.toHaveBeenCalled() expect(browserB.overwriteCommand).toHaveBeenCalledWith('execute', expect.any(Function)) }) + + describe('command and result registration on the CLI flow', () => { + let getInstanceSpy: ReturnType | undefined + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + const registeredEvents = async (framework: string) => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + const cliBrowser = { on: vi.fn(), sessionId: 's1', capabilities: {}, config: {}, execute: vi.fn(), executeScript: vi.fn() } as any + const service = new BrowserstackService({} as any, [{}] as any, { user: 'foo', key: 'bar', framework, capabilities: {} } as any) + await service.before(service['_config'] as any, [], cliBrowser) + delete process.env.BROWSERSTACK_OBSERVABILITY + return vi.mocked(cliBrowser.on).mock.calls.map(([event]: [string]) => event) + } + + it('registers command and result for jasmine', async () => { + const events = await registeredEvents('jasmine') + expect(events).toContain('command') + expect(events).toContain('result') + }) + + it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { + const events = await registeredEvents(framework) + expect(events).toContain('result') + expect(events).not.toContain('command') + }) + }) }) describe('beforeHook', () => { @@ -1154,6 +1194,39 @@ describe('beforeTest', () => { ) }) }) + + describe('on the CLI flow', () => { + let getInstanceSpy: ReturnType | undefined + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + const runBeforeTest = async (framework: string, test: Record) => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => ({ trackEvent }), + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + const service = new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + const annotate = vi.spyOn(service as any, '_setAnnotation').mockResolvedValue(undefined) + const getState = vi.spyOn(TestFramework, 'getState').mockReturnValue('spec-uuid') + await service.beforeTest(test as any) + getState.mockRestore() + return { annotate, trackEvent } + } + + it.each([ + ['jasmine', jasmineLastSpec], + ['mocha', { title: 't', parent: 'suite' }], + ])('leaves the %s test annotation to AutomateModule and still sends TEST/PRE', async (framework, test) => { + const { annotate, trackEvent } = await runBeforeTest(framework, test as Record) + expect(annotate).not.toHaveBeenCalled() + expect(trackEvent).toHaveBeenCalledWith(TestFrameworkState.TEST, HookState.PRE, expect.objectContaining({ test })) + }) + }) }) describe('afterTest', () => { @@ -2638,6 +2711,80 @@ describe('beforeHook (CLI hook reporting)', () => { expect(service['_insightsHandler']!.beforeHook).toHaveBeenCalledTimes(1) }) + + describe('classification by framework', () => { + const mochaHook = (title: string) => ({ title, ctx: { test: { parent: { title: 'suite', tests: [], suites: [] } } } }) as any + const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + } + const makeService = (framework: string) => new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + + it('classifies jasmine hooks by hookName, without touching the missing title', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const getHookTypeSpy = vi.spyOn(utils, 'getHookType') + const service = makeService('jasmine') + + for (const hookName of ['beforeAll', 'beforeEach']) { + await service.beforeHook(jasmineLastSpec, {}, hookName) + } + for (const hookName of ['afterEach', 'afterAll']) { + await service.afterHook(jasmineLastSpec, {}, { passed: true } as any, hookName) + } + + expect(getHookTypeSpy).not.toHaveBeenCalled() + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.AFTER_EACH, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + getHookTypeSpy.mockRestore() + }) + + it('sends nothing for a jasmine hook without a hookName, and does not throw', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const service = makeService('jasmine') + + await expect(service.beforeHook({} as any, {})).resolves.toBeUndefined() + await expect(service.afterHook({} as any, {}, { passed: true } as any)).resolves.toBeUndefined() + expect(trackEvent).not.toHaveBeenCalled() + }) + + it('never runs the mocha skip cascade for a failed jasmine hook, but still records the hook failure', async () => { + cliWith({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('jasmine') + + await service.afterHook({ ...jasmineLastSpec, ctx: { test: { parent: {} } } }, {}, { passed: false, error: new Error('beforeAll failed') } as any, 'beforeAll') + + expect(cascade).not.toHaveBeenCalled() + expect(service['_hookFailReasons']).toEqual(['beforeAll failed']) + cascade.mockRestore() + }) + + it('keeps mocha on its title classification and skip cascade', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('mocha') + + await service.beforeHook(mochaHook('"before each" hook for "t"'), {}, 'beforeEach') + await service.afterHook(mochaHook('"before all" hook for "t"'), {}, { passed: false, error: new Error('x') } as any, 'afterAll') + + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + ]) + expect(cascade).toHaveBeenCalledTimes(1) + cascade.mockRestore() + }) + }) }) describe('_isAppAutomate honors skipAppOverride', () => { @@ -3126,3 +3273,65 @@ describe('_cucumberTestResult failure reason adjacent', () => { expect(result.error?.message).toBe('Unknown Error') }) }) + +describe('after hands AutomateModule the jasmine session verdict inputs', () => { + // describe('after') swaps the prototype method for a stand-in; these cases need the real one + const after = BrowserstackService.prototype.after + let trackEvent: ReturnType + let getInstanceSpy: ReturnType | undefined + + const makeService = (framework: string) => new BrowserstackService( + { testObservability: false, setSessionStatus: true } as never, + [] as never, + { user: 'foo', key: 'bar', framework } as never + ) + const executeArgs = () => trackEvent.mock.calls.find(([state, hook]) => state === AutomationFrameworkState.EXECUTE && hook === HookState.POST)?.[2] + + beforeEach(() => { + trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent }), + modules: {} + } as never) + ;(PerformanceTester as unknown as Record).stopAndGenerate = vi.fn().mockResolvedValue(undefined) + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + delete (PerformanceTester as unknown as Record).stopAndGenerate + }) + + it.each(['mocha', 'cucumber'])('sends nothing extra for %s', async (framework) => { + const service = makeService(framework) + service['_specsRan'] = true + service['_failReasons'] = ['boom'] + + await after.call(service, 1) + + expect(executeArgs()).toEqual({}) + }) + + it('sends the runner result and a copy of the tracked reasons for jasmine', async () => { + const service = makeService('jasmine') + service['_specsRan'] = true + service['_failReasons'] = ['beforeEach hook failure', 'boom'] + service['_pureTestFailReasons'] = ['boom'] + service['_hookFailReasons'] = ['beforeEach hook failure'] + + await after.call(service, 2) + + const args = executeArgs() as { sessionVerdictInputs: Record } + expect(args).toEqual({ + sessionVerdictInputs: { + result: 2, + specsRan: true, + failReasons: ['beforeEach hook failure', 'boom'], + pureTestFailReasons: ['boom'], + hookFailReasons: ['beforeEach hook failure'], + } + }) + expect(args.sessionVerdictInputs.failReasons).not.toBe(service['_failReasons']) + }) +}) diff --git a/packages/browserstack-service/tests/skipAppOverride.test.ts b/packages/browserstack-service/tests/skipAppOverride.test.ts index 5b175848..f65e36e8 100644 --- a/packages/browserstack-service/tests/skipAppOverride.test.ts +++ b/packages/browserstack-service/tests/skipAppOverride.test.ts @@ -89,7 +89,9 @@ describe('NOT_ALLOWED_KEYS_IN_CAPS cloud-leak strip', () => { expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('skipAppOverride') }) - it('includes preferScenarioName so it is never forwarded to bstack:options', () => { + it('includes the SDK-only options the binary round-trips into caps on the CLI flow', () => { expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('preferScenarioName') + expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('sessionNamePrependTopLevelSuiteTitle') + expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('sessionNameOmitTestTitle') }) })