From e19eebf4512880df9bb8050da98ae89fd7f455bb Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 08:41:09 +0530 Subject: [PATCH 01/15] feat(browserstack-service): route wdio jasmine to the CLI flow (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../browserstack-service/src/cli/cliUtils.ts | 2 +- .../frameworks/wdioJasmineTestFramework.ts | 16 ++++++ .../browserstack-service/src/cli/index.ts | 7 ++- .../tests/cli/wdioJasmineGate.test.ts | 57 +++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts create mode 100644 packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 3c6e5c05..6fbe10bf 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -47,7 +47,7 @@ const CLI_DOWNLOAD_TMP_SUFFIX = '.zip' export class CLIUtils { static automationFrameworkDetail = {} static testFrameworkDetail = {} - static CLISupportedFrameworks = ['mocha', 'cucumber'] + static CLISupportedFrameworks = ['mocha', 'cucumber', 'jasmine'] static isDevelopmentEnv() { return process.env.BROWSERSTACK_CLI_ENV === 'development' diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts new file mode 100644 index 00000000..68524b06 --- /dev/null +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -0,0 +1,16 @@ +import TestFramework from './testFramework.js' +import { BStackLogger as logger } from '../cliLogger.js' + +/** + * CLI test framework for `framework: 'jasmine'` under WebdriverIO. + * + * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), + * hook taxonomy (`hookName`) and skip/exclude states differ from mocha's, and inheriting mocha's + * event model would report jasmine specs through the wrong runner's semantics. + */ +export default class WdioJasmineTestFramework extends TestFramework { + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { + super(testFrameworks, testFrameworkVersions, binSessionId) + logger.debug('WdioJasmineTestFramework: constructed') + } +} diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 444ec6d7..735ccc6d 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -17,6 +17,7 @@ import type { Options } from '@wdio/types' import TestOpsConfig from '../testOps/testOpsConfig.js' import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from './frameworks/wdioJasmineTestFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' @@ -48,7 +49,7 @@ export class BrowserstackCLI { modulesLoaded = false binSessionId: string | null = null modules: Record = {} - testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|null = null + testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|WdioJasmineTestFramework|null = null cliParams: Record | null = null automationFramework: WdioAutomationFramework|null = null SDK_CLI_BIN_PATH: string | null = null @@ -555,6 +556,10 @@ export class BrowserstackCLI { this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) return } + if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-jasmine') { + this.testFramework = new WdioJasmineTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + return + } // An unmatched name leaves testFramework null, and every CLI event then no-ops with no // error of any kind. Name it so the silence is diagnosable. this.logger.error(`setupTestFramework: no CLI test framework registered for name=${testFrameworkDetail.name}; test events will not be tracked`) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts new file mode 100644 index 00000000..424744d9 --- /dev/null +++ b/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi, afterEach } from 'vitest' +import * as bstackLogger from '../../src/bstackLogger.js' + +import { BrowserstackCLI } from '../../src/cli/index.js' +import { CLIUtils } from '../../src/cli/cliUtils.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +describe('CLIUtils.checkCLISupportedFrameworks', () => { + it.each(['mocha', 'cucumber', 'jasmine'])('routes %s to the CLI flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(true) + }) + + it.each([undefined, 'WebdriverIO-jasmine', 'Jasmine', 'unknown'])('keeps %s on the legacy flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(false) + }) +}) + +describe('BrowserstackCLI.setupTestFramework', () => { + const cli = BrowserstackCLI.getInstance() + + afterEach(() => { + vi.restoreAllMocks() + cli.testFramework = null + }) + + const setup = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + cli.setupTestFramework() + return cli.getTestFramework() + } + + it('constructs the jasmine framework for WebdriverIO-jasmine', () => { + const framework = setup('WebdriverIO-jasmine') + expect(framework).toBeInstanceOf(WdioJasmineTestFramework) + expect(framework).not.toBeInstanceOf(WdioMochaTestFramework) + expect(framework).toBeInstanceOf(TestFramework) + expect(framework!.getTestFrameworks()).toEqual(['WebdriverIO-jasmine']) + expect(framework!.getTestFrameworksVersions()).toEqual({ 'WebdriverIO-jasmine': '9.0.0' }) + }) + + it('still constructs the mocha framework for WebdriverIO-mocha', () => { + expect(setup('WebdriverIO-mocha')).toBeInstanceOf(WdioMochaTestFramework) + }) + + it('still constructs the cucumber framework for WebdriverIO-cucumber', () => { + expect(setup('WebdriverIO-cucumber')).toBeInstanceOf(WdioCucumberTestFramework) + }) + + it('leaves an unknown name unregistered', () => { + expect(setup('WebdriverIO-unknown')).toBeNull() + }) +}) From 3ae9e25ce118bf0f5daa2432cd24c45268a109b2 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 09:54:51 +0530 Subject: [PATCH 02/15] feat(browserstack-service): add WdioJasmineTestFramework and reporter CLI feed (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 363 +++++++++++++++++- packages/browserstack-service/src/reporter.ts | 52 +++ .../cli/wdioJasmineTestFramework.test.ts | 295 ++++++++++++++ .../tests/reporter.jasmineCli.test.ts | 188 +++++++++ 4 files changed, 896 insertions(+), 2 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts create mode 100644 packages/browserstack-service/tests/reporter.jasmineCli.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 68524b06..45dde1af 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -1,16 +1,375 @@ +import path from 'node:path' +import util from 'node:util' +import { v4 as uuidv4 } from 'uuid' +import type { Frameworks } from '@wdio/types' +import type { HookStats, TestStats } from '@wdio/reporter' + import TestFramework from './testFramework.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' +import { HookState } from '../states/hookState.js' +import TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import TrackedInstance from '../instances/trackedInstance.js' +import { CLIUtils } from '../cliUtils.js' +import { TestFrameworkConstants } from './constants/testFrameworkConstants.js' import { BStackLogger as logger } from '../cliLogger.js' +import { getHookType, getTestTags, removeAnsiColors } from '../../util.js' +import { shouldProcessEventForTesthub } from '../../testHub/utils.js' +import { TEST_ANALYTICS_ID } from '../../constants.js' + +/** What the reporter knows at each event and the framework cannot see: its suite stack and suite file. */ +export interface JasmineSuiteContext { + scopes: string[] + suiteFile?: string +} + +interface FailureFields { + failure?: Array<{ backtrace: string[] }> + failureReason?: string + failureType?: string | null +} /** * CLI test framework for `framework: 'jasmine'` under WebdriverIO. * * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), - * hook taxonomy (`hookName`) and skip/exclude states differ from mocha's, and inheriting mocha's - * event model would report jasmine specs through the wrong runner's semantics. + * hook taxonomy and skip/exclude states differ from mocha's. + * + * Test and hook states come from the WDIO reporter (`reporter.ts`), the only jasmine source that + * sees pending, focused-out and beforeAll-failed specs. Reporter hooks are not awaited by WDIO, so + * every reporter event is applied and dispatched through one ordered queue. */ export default class WdioJasmineTestFramework extends TestFramework { + // Same data keys WdioMochaTestFramework uses: TestHubModule's LOG arm reads them through + // WdioMochaTestFramework.getLogEntries/clearLogs. + static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' + static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + static KEY_TEST_RETRIES = 'test_retries' + static KEY_TEST_DURATION = 'test_duration' + static KEY_HOOK_IDENTIFIER = 'hook_identifier' + static KEY_HOOK_SCOPE = 'hook_scope' + static KEY_HOOK_SCOPES = 'hook_scopes' + static KEY_HOOK_DURATION = 'hook_duration' + static KEY_HOOK_FAILURE = 'hook_failure' + static KEY_HOOK_FAILURE_TYPE = 'hook_failure_type' + static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' + + static #pendingEvents = 0 + + #specInstances = new Map() + // beforeAll/afterAll never overlap, so a hook's uid (its title) is unique while it is open. + #hookInstances = new Map() + #openHook: TestFrameworkInstance | null = null + #lastSpec: TestFrameworkInstance | null = null + #queue: Promise = Promise.resolve() + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { super(testFrameworks, testFrameworkVersions, binSessionId) logger.debug('WdioJasmineTestFramework: constructed') } + + /** True when no reporter event is still waiting to be dispatched (the reporter's `isSynchronised`). */ + static isIdle() { + return WdioJasmineTestFramework.#pendingEvents === 0 + } + + /** + * Service hooks (beforeTest/afterTest and friends) also call this. Their TEST/INIT_TEST/LOG_REPORT + * states would duplicate what the reporter already dispatched, so they are absorbed; waiting for the + * queue first leaves the tracked instance pointing at the spec the service is running. + */ + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + if (testFrameworkState === TestFrameworkState.LOG) { + this.onReporterLog(args.logEntry as Record) + return + } + await this.#queue + logger.debug(`WdioJasmineTestFramework: absorbed service event ${testFrameworkState}/${hookState}`) + } + + onReporterTestStart(testStats: TestStats, context: JasmineSuiteContext) { + try { + logger.debug(`WdioJasmineTestFramework: test start uid=${testStats.uid}`) + const instance = this.#createInstance() + const scopes = [...context.scopes] + const fullTitle = testStats.fullTitle + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + [TestFrameworkConstants.KEY_TEST_ID]: fullTitle, + [TestFrameworkConstants.KEY_TEST_NAME]: testStats.title, + [TestFrameworkConstants.KEY_TEST_SCOPE]: fullTitle, + [TestFrameworkConstants.KEY_TEST_SCOPES]: scopes, + [TestFrameworkConstants.KEY_TEST_TAGS]: getTestTags(testStats as unknown as Frameworks.Test, scopes), + ...this.#filePaths(context.suiteFile), + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), + [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, + [WdioJasmineTestFramework.KEY_TEST_RETRIES]: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, + }) + this.#specInstances.set(testStats.uid, instance) + // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. + TestFramework.setTrackedInstance(instance.getContext(), instance) + + const args = { test: this.#specArg(testStats, context), suiteTitle: this.#suiteTitle(testStats) } + this.#enqueue('TEST/PRE', async () => { + this.#lastSpec = instance + process.env[TEST_ANALYTICS_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) + await this.#emit(instance, TestFrameworkState.TEST, HookState.PRE, args) + }) + return TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string + } catch (error) { + logger.error(`WdioJasmineTestFramework: test start failed: ${util.format(error)}`) + } + } + + onReporterTestEnd(testStats: TestStats, context: JasmineSuiteContext) { + try { + const instance = this.#specInstances.get(testStats.uid) + if (!instance) { + logger.debug(`WdioJasmineTestFramework: no instance for test end uid=${testStats.uid}`) + return + } + this.#specInstances.delete(testStats.uid) + + const state = testStats.state + const error = testStats.error + let result: string = state + let failure: FailureFields = {} + if (state === 'failed') { + result = (error && error.message && error.message.includes('sync skip; aborting execution')) ? 'ignore' : 'failed' + if (error) { + failure = this.#failureFields(error) + } + } + const entries: Record = { + [TestFrameworkConstants.KEY_TEST_RESULT]: result, + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: this.#iso(testStats.end), + [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), + } + if (failure.failure) { + entries[TestFrameworkConstants.KEY_TEST_FAILURE] = failure.failure + entries[TestFrameworkConstants.KEY_TEST_FAILURE_REASON] = failure.failureReason + entries[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE] = failure.failureType + } + const testResult = { + passed: state === 'passed', + skipped: state === 'skipped', + error, + duration: testStats._duration, + retries: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, + } + const args = { test: this.#specArg(testStats, context), result: testResult, suiteTitle: this.#suiteTitle(testStats) } + this.#enqueue('TEST/POST', async () => { + instance.updateMultipleEntries(entries) + await this.#emit(instance, TestFrameworkState.TEST, HookState.POST, args) + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: test end failed: ${util.format(error)}`) + } + } + + onReporterHookStart(hookStats: HookStats, context: JasmineSuiteContext) { + try { + const title = hookStats.title + const key = getHookType(String(title ?? '').toLowerCase()) + const hookFrameworkState = WdioJasmineTestFramework.#hookState(key) + if (!hookFrameworkState || context.scopes.length === 0) { + logger.debug(`WdioJasmineTestFramework: hook start not reported title=${title} scopes=${context.scopes.length}`) + return + } + const scopes = [...context.scopes] + const instance = this.#createInstance() + const hookId = uuidv4() + const hook: Record = { + key, + [TestFrameworkConstants.KEY_HOOK_ID]: hookId, + [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: this.#iso(hookStats.start), + [TestFrameworkConstants.KEY_HOOK_LOGS]: [], + [TestFrameworkConstants.KEY_HOOK_NAME]: title, + [WdioJasmineTestFramework.KEY_HOOK_IDENTIFIER]: `${title} for ${scopes[scopes.length - 1]}`, + [WdioJasmineTestFramework.KEY_HOOK_SCOPE]: `${scopes[0]} - ${title}`, + [WdioJasmineTestFramework.KEY_HOOK_SCOPES]: scopes, + [WdioJasmineTestFramework.KEY_HOOK_DURATION]: hookStats._duration, + ...this.#filePaths(context.suiteFile), + } + instance.updateData(TestFrameworkConstants.KEY_HOOK_ID, hookId) + this.#hookInstances.set(hookStats.uid, instance) + + const emitHook = shouldProcessEventForTesthub('HookRunStarted') + this.#enqueue(`${key}/PRE`, async () => { + this.#openHook = instance + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map + hooksStarted.set(key, [hook]) + instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_STARTED, key) + if (emitHook) { + await this.#emit(instance, hookFrameworkState, HookState.PRE, { hook: hookStats }) + } + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: hook start failed: ${util.format(error)}`) + } + } + + onReporterHookEnd(hookStats: HookStats) { + try { + const instance = this.#hookInstances.get(hookStats.uid) + if (!instance) { + return + } + this.#hookInstances.delete(hookStats.uid) + const key = getHookType(String(hookStats.title ?? '').toLowerCase()) + const hookFrameworkState = WdioJasmineTestFramework.#hookState(key)! + + const state = hookStats.state + const error = hookStats.error + let result = state as string | undefined + let failure: FailureFields = {} + if (state === 'failed') { + result = (error && error.message && error.message.includes('sync skip; aborting execution')) ? 'ignore' : 'failed' + if (error) { + failure = this.#failureFields(error) + } + } + const endedAt = this.#iso(hookStats.end) + const duration = hookStats._duration + const emitHook = shouldProcessEventForTesthub('HookRunFinished') + this.#enqueue(`${key}/POST`, async () => { + if (this.#openHook === instance) { + this.#openHook = null + } + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map[]> + const hooksFinished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map[]> + const hook = hooksStarted.get(key)?.pop() + if (!hook) { + return + } + if (result) { + hook[TestFrameworkConstants.KEY_HOOK_RESULT] = result + } + hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = endedAt + hook[WdioJasmineTestFramework.KEY_HOOK_DURATION] = duration + if (failure.failure) { + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE] = failure.failure + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE_REASON] = failure.failureReason + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE_TYPE] = failure.failureType + } + hooksFinished.set(key, [hook]) + instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_FINISHED, key) + if (emitHook) { + await this.#emit(instance, hookFrameworkState, HookState.POST, { hook: hookStats, result: { passed: result === 'passed', error } }) + } + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: hook end failed: ${util.format(error)}`) + } + } + + /** Console logs and screenshots: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. */ + onReporterLog(logEntry: Record | undefined) { + try { + if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { + return + } + this.#enqueue('LOG/POST', async () => { + const instance = this.#openHook ?? this.#lastSpec + if (!instance) { + return + } + const logRecord: Record = { + kind: logEntry.kind ?? TestFrameworkConstants.KIND_LOG, + message: Buffer.from(String(logEntry.message ?? '')), + level: logEntry.level, + timestamp: logEntry.timestamp, + } + if (instance === this.#openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) + } + const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] + entries.push(logRecord) + await this.#emit(instance, TestFrameworkState.LOG, HookState.POST, { logEntry }) + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: log failed: ${util.format(error)}`) + } + } + + #createInstance() { + const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + TestFrameworkState.NONE, + HookState.NONE + ) + const frameworkName = this.getTestFrameworks()[0] + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + }) + return instance + } + + async #emit(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + TestFramework.setTrackedInstance(instance.getContext(), instance) + this.updateInstanceState(instance, testFrameworkState, hookState) + args.instance = instance + await this.runHooks(instance, testFrameworkState, hookState, args) + } + + #enqueue(label: string, step: () => Promise) { + WdioJasmineTestFramework.#pendingEvents++ + this.#queue = this.#queue.then(step).catch((error) => { + logger.error(`WdioJasmineTestFramework: ${label} failed: ${util.format(error)}`) + }).finally(() => { + WdioJasmineTestFramework.#pendingEvents-- + }) + } + + /** The binary re-bases `test_file_path` itself, so it must be absolute. */ + #filePaths(suiteFile: string | undefined) { + const absolute = suiteFile ? path.resolve(suiteFile) : undefined + return { + [TestFrameworkConstants.KEY_TEST_FILE_PATH]: absolute, + [TestFrameworkConstants.KEY_TEST_LOCATION]: absolute ? path.relative(process.cwd(), absolute) : undefined, + } + } + + #failureFields(error: Error): FailureFields { + return { + failure: [{ backtrace: [removeAnsiColors(error.message), removeAnsiColors(error.stack || '')] }], + failureReason: removeAnsiColors(error.message), + failureType: error.message === null ? null : error.message.toString().match(/AssertionError/) ? 'AssertionError' : 'UnhandledError', + } + } + + /** SpecResult-shaped, like the service hooks' `test`: jasmine has no `title`. */ + #specArg(testStats: TestStats, context: JasmineSuiteContext) { + return { description: testStats.title, fullName: testStats.fullTitle, file: context.suiteFile } as unknown as Frameworks.Test + } + + /** The describe chain, derived the way `service.beforeTest` does for jasmine. */ + #suiteTitle(testStats: TestStats) { + const fullName = testStats.fullTitle || '' + return fullName.slice(0, fullName.indexOf(testStats.title || '') - 1) + } + + #iso(date: Date | undefined) { + return date ? date.toISOString() : undefined + } + + static #hookState(key: string) { + if (key === 'BEFORE_ALL') { + return TestFrameworkState.BEFORE_ALL + } + if (key === 'AFTER_ALL') { + return TestFrameworkState.AFTER_ALL + } + return null + } } diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index 68d77738..3e684124 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -5,6 +5,8 @@ import WDIOReporter from '@wdio/reporter' import type { Options, Frameworks } from '@wdio/types' import { BrowserstackCLI } from './cli/index.js' import { reportSkippedTest, resolveSpecFile } from './cli/skipReporter.js' +import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' +import type { JasmineSuiteContext } from './cli/frameworks/wdioJasmineTestFramework.js' import * as url from 'node:url' import { v4 as uuidv4 } from 'uuid' @@ -72,6 +74,10 @@ class _TestReporter extends WDIOReporter { } public async appendTestItemLog(stdLog: StdLog) { + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.onReporterLog(stdLog as unknown as Record) + return + } if (this._currentHook.uuid && !this._currentHook.finished) { stdLog.hook_run_uuid = this._currentHook.uuid } else if (_TestReporter.currentTest.uuid) { @@ -134,6 +140,29 @@ class _TestReporter extends WDIOReporter { this._suites.pop() } + /** + * WDIO waits for reporters to synchronise before the worker exits; the CLI jasmine feed dispatches + * asynchronously, so hold the worker until it has drained. Always true on every other path. + */ + get isSynchronised() { + return WdioJasmineTestFramework.isIdle() + } + + /** On the CLI flow jasmine's test/hook/log events feed the framework tracker, never the legacy Listener. */ + isCliJasmine() { + return this._config?.framework === 'jasmine' && BrowserstackCLI.getInstance().isRunning() + } + + cliJasmineFramework() { + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioJasmineTestFramework ? framework : null + } + + jasmineSuiteContext(): JasmineSuiteContext { + const suiteFile = this._suiteName || (this.specs?.length > 0 ? this.specs[this.specs.length - 1]?.replace('file:', '') : undefined) + return { scopes: this._suites.map(s => s.title), suiteFile } + } + needToSendData(testType?: string, event?: string) { if (!this._observability) {return false} @@ -158,6 +187,10 @@ class _TestReporter extends WDIOReporter { } testStats.end ||= new Date() + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.onReporterTestEnd(testStats, this.jasmineSuiteContext()) + return + } this.listener.testFinished(await this.getRunData(testStats, 'TestRunFinished')) } @@ -168,6 +201,14 @@ class _TestReporter extends WDIOReporter { if (testStats.fullTitle === '') { return } + if (this.isCliJasmine()) { + const cliUuid = this.cliJasmineFramework()?.onReporterTestStart(testStats, this.jasmineSuiteContext()) + if (cliUuid) { + _TestReporter.currentTest.uuid = cliUuid + _TestReporter._tests[testStats.fullTitle] = { uuid: cliUuid } + } + return + } const uuid = uuidv4() _TestReporter.currentTest.uuid = uuid @@ -181,6 +222,10 @@ class _TestReporter extends WDIOReporter { if (!this.needToSendData('hook', 'start')) { return } + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.onReporterHookStart(hookStats, this.jasmineSuiteContext()) + return + } const identifier = this.getHookIdentifier(hookStats) const hookId = uuidv4() @@ -196,6 +241,13 @@ class _TestReporter extends WDIOReporter { if (!this.needToSendData('hook', 'end')) { return } + if (this.isCliJasmine()) { + if (!hookStats.state && !hookStats.error) { + hookStats.state = 'passed' + } + this.cliJasmineFramework()?.onReporterHookEnd(hookStats) + return + } const identifier = this.getHookIdentifier(hookStats) if (_TestReporter._tests[identifier]) { _TestReporter._tests[identifier].finishedAt = (new Date()).toISOString() diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts new file mode 100644 index 00000000..8eec475f --- /dev/null +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -0,0 +1,295 @@ +import path from 'node:path' +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +import * as bstackLogger from '../../src/bstackLogger.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../src/cli/states/hookState.js' +import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +const SUITE_FILE = path.join(process.cwd(), 'test/p2/nested.spec.js') +const BEFORE_ALL_REASON = 'Not run because a beforeAll function failed. The beforeAll failure will be reported on the suite that caused it.' + +interface Dispatch { + state: State + hook: State + data: Record + args: Record +} + +const snapshot = (instance: TestFrameworkInstance) => + JSON.parse(JSON.stringify(Object.fromEntries(instance.getAllData()), (_k, v) => v instanceof Map ? Object.fromEntries(v) : v)) + +const testStats = (overrides: Record = {}) => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + end: undefined as Date | undefined, + _duration: 0, + retries: 0, + state: 'pending', + ...overrides, +}) + +const hookStats = (title: string, overrides: Record = {}) => ({ + type: 'hook', + uid: title, + title, + start: new Date('2026-09-25T15:34:08.170Z'), + end: undefined as Date | undefined, + _duration: 0, + ...overrides, +}) + +const context = (scopes = ['Nested outer']) => ({ scopes, suiteFile: SUITE_FILE }) + +describe('WdioJasmineTestFramework', () => { + let framework: WdioJasmineTestFramework + let dispatches: Dispatch[] + + const drain = () => framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, {}) + + beforeEach(() => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') + dispatches = [] + vi.spyOn(framework, 'runHooks').mockImplementation(async (instance, state, hook, args) => { + dispatches.push({ state, hook, data: snapshot(instance), args: args as Record }) + }) + }) + + afterEach(() => { + vi.restoreAllMocks() + delete process.env.BROWSERSTACK_OBSERVABILITY + }) + + it('sends legacy identity for a spec, char-for-char with the CP0 TestRun', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await drain() + + expect(dispatches).toHaveLength(1) + const [{ state, hook, data, args }] = dispatches + expect([state, hook]).toEqual([TestFrameworkState.TEST, HookState.PRE]) + expect(data.test_name).toBe('outer passing test') + expect(data.test_id).toBe('Nested outer outer passing test') + expect(data.test_scope).toBe('Nested outer outer passing test') + expect(data.test_scopes).toEqual(['Nested outer']) + expect(data.test_tags).toEqual([]) + expect(data.test_file_path).toBe(SUITE_FILE) + expect(data.test_location).toBe('test/p2/nested.spec.js') + expect(data.test_framework_name).toBe('WebdriverIO-jasmine') + expect(data.test_retries).toEqual({ limit: 0, attempts: 0 }) + expect(data.test_started_at).toBe('2026-09-25T15:34:12.259Z') + expect(data.test_duration).toBe(0) + expect(data.test_result).toBe('pending') + expect('test_code' in data).toBe(false) + expect(args.suiteTitle).toBe('Nested outer') + expect(args.test).toEqual({ description: 'outer passing test', fullName: 'Nested outer outer passing test', file: SUITE_FILE }) + }) + + it('keeps @tags from describes and the spec, sigil included', async () => { + framework.onReporterTestStart(testStats({ title: 'tagged @smoke', fullTitle: 'Outer @regression tagged @smoke' }) as any, context(['Outer @regression'])) + await drain() + expect(dispatches[0].data.test_tags).toEqual(['@regression', '@smoke']) + }) + + it('finishes a passed spec on the same uuid with reporter timing and the result timestamp', async () => { + const stats = testStats() + framework.onReporterTestStart(stats as any, context()) + framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date('2026-09-25T15:34:16.645Z'), _duration: 4386 } as any, context()) + await drain() + + expect(dispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) + const [start, finish] = dispatches + expect(finish.data.test_uuid).toBe(start.data.test_uuid) + expect(finish.data.test_result).toBe('passed') + expect(finish.data.test_started_at).toBe('2026-09-25T15:34:12.259Z') + expect(finish.data.test_ended_at).toBe('2026-09-25T15:34:16.645Z') + expect(finish.data.test_duration).toBe(4386) + expect(typeof finish.data.test_result_at).toBe('string') + expect('test_failure' in finish.data).toBe(false) + expect(finish.args.result).toMatchObject({ passed: true, skipped: false }) + }) + + it('reports a pending spec as Start + Finish(skipped)', async () => { + framework.onReporterTestStart(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test' }) as any, context(['Pending suite'])) + // @wdio/reporter replaces the TestStats object on test:pending; only the uid carries over + framework.onReporterTestEnd(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test', state: 'skipped', end: new Date() }) as any, context(['Pending suite'])) + await drain() + + expect(dispatches).toHaveLength(2) + expect(dispatches[1].data.test_result).toBe('skipped') + expect(dispatches[1].data.test_duration).toBe(0) + expect(dispatches[1].args.result).toMatchObject({ passed: false, skipped: true }) + }) + + it('keeps jasmine\'s exact reason on a beforeAll-failed child', async () => { + const error = { message: BEFORE_ALL_REASON, stack: '' } + framework.onReporterTestStart(testStats({ uid: 'child0' }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'child0', state: 'failed', error, end: new Date() }) as any, context()) + await drain() + + const finish = dispatches[1].data + expect(finish.test_result).toBe('failed') + expect(finish.test_failure_reason).toBe(BEFORE_ALL_REASON) + expect(finish.test_failure_type).toBe('UnhandledError') + expect(finish.test_failure).toEqual([{ backtrace: [BEFORE_ALL_REASON, ''] }]) + }) + + it('maps an AssertionError message to failure_type AssertionError', async () => { + framework.onReporterTestStart(testStats() as any, context()) + framework.onReporterTestEnd(testStats({ state: 'failed', error: { message: 'AssertionError: nope', stack: 'at x' } }) as any, context()) + await drain() + expect(dispatches[1].data.test_failure_type).toBe('AssertionError') + }) + + it('gives overlapping specs distinct instances and closes each on its own uuid', async () => { + framework.onReporterTestStart(testStats({ uid: 'a' }) as any, context()) + framework.onReporterTestStart(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b' }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'a', state: 'passed', end: new Date() }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b', state: 'passed', end: new Date() }) as any, context()) + await drain() + + const uuidOf = (i: number) => dispatches[i].data.test_uuid + expect(uuidOf(0)).not.toBe(uuidOf(1)) + expect(uuidOf(2)).toBe(uuidOf(0)) + expect(uuidOf(3)).toBe(uuidOf(1)) + expect(dispatches[2].data.test_id).toBe('Nested outer outer passing test') + }) + + it('reports beforeAll/afterAll with legacy hook identity and no test linkage', async () => { + const before = hookStats('"before all" hook') + framework.onReporterHookStart(before as any, context()) + framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date('2026-09-25T15:34:12.257Z'), _duration: 4087 } as any) + const after = hookStats('"after all" hook') + framework.onReporterHookStart(after as any, context(['Nested outer', 'Nested middle'])) + framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) + await drain() + + expect(dispatches.map(d => [d.state, d.hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.PRE], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + const started = (dispatches[0].data.test_hooks_started as Record[]>).BEFORE_ALL[0] + expect(started).toMatchObject({ + key: 'BEFORE_ALL', + hook_name: '"before all" hook', + hook_identifier: '"before all" hook for Nested outer', + hook_scope: 'Nested outer - "before all" hook', + hook_scopes: ['Nested outer'], + hook_result: 'pending', + event_started_at: '2026-09-25T15:34:08.170Z', + test_file_path: SUITE_FILE, + }) + const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] + expect(finished).toMatchObject({ hook_result: 'passed', event_ended_at: '2026-09-25T15:34:12.257Z', hook_duration: 4087 }) + expect(finished.hook_id).toBe(started.hook_id) + expect('hook_failure' in finished).toBe(false) + expect('test_uuid' in dispatches[0].data).toBe(false) + + const afterStarted = (dispatches[2].data.test_hooks_started as Record[]>).AFTER_ALL[0] + expect(afterStarted.hook_identifier).toBe('"after all" hook for Nested middle') + expect(afterStarted.hook_scope).toBe('Nested outer - "after all" hook') + }) + + it('carries failure fields on a failed beforeAll', async () => { + const before = hookStats('"before all" hook') + framework.onReporterHookStart(before as any, context()) + framework.onReporterHookEnd({ ...before, state: 'failed', error: { message: 'boom', stack: 'Error: boom' }, end: new Date() } as any) + await drain() + const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] + expect(finished).toMatchObject({ + hook_result: 'failed', + hook_failure: [{ backtrace: ['boom', 'Error: boom'] }], + hook_failure_reason: 'boom', + hook_failure_type: 'UnhandledError', + }) + }) + + it('never reports each-hooks, or a hook outside any describe', async () => { + framework.onReporterHookStart(hookStats('"before each" hook') as any, context()) + framework.onReporterHookStart(hookStats('"before all" hook') as any, context([])) + await drain() + expect(dispatches).toHaveLength(0) + }) + + it('suppresses hook and log families in an accessibility-only run, like the legacy Listener', async () => { + delete process.env.BROWSERSTACK_OBSERVABILITY + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + try { + const before = hookStats('"before all" hook') + framework.onReporterHookStart(before as any, context()) + framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date() } as any) + framework.onReporterTestStart(testStats() as any, context()) + framework.onReporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) + await drain() + expect(dispatches.map(d => d.state)).toEqual([TestFrameworkState.TEST]) + } finally { + delete process.env.BROWSERSTACK_ACCESSIBILITY + } + }) + + it('absorbs service-hook states instead of re-dispatching them, leaving the spec tracked', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: {} }) + const tracked = TestFramework.getTrackedInstance() + expect(TestFramework.getState(tracked, 'test_uuid')).toBe(dispatches[0].data.test_uuid) + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: {} }) + await framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test: {}, result: {} }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: {}, result: {} }) + expect(dispatches).toHaveLength(1) + }) + + it('attributes logs to an open all-hook, else to the last-started spec', async () => { + framework.onReporterTestStart(testStats() as any, context()) + framework.onReporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) + const after = hookStats('"after all" hook') + framework.onReporterHookStart(after as any, context()) + framework.onReporterLog({ level: 'INFO', message: 'in hook', timestamp: 't2', kind: 'TEST_LOG' }) + framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) + framework.onReporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) + await drain() + + const logs = dispatches.filter(d => d.state === TestFrameworkState.LOG) + expect(logs).toHaveLength(3) + const specUuid = dispatches[0].data.test_uuid + expect(logs[0].data.test_uuid).toBe(specUuid) + expect((logs[0].data.test_logs as Record[]).at(-1)).not.toHaveProperty('hook_id') + const hookLog = (logs[1].data.test_logs as Record[]).at(-1)! + expect(hookLog.hook_id).toBe((dispatches.find(d => d.state === TestFrameworkState.AFTER_ALL)!.data.test_hooks_started as Record[]>).AFTER_ALL[0].hook_id) + expect(logs[2].data.test_uuid).toBe(specUuid) + }) + + it('keeps a screenshot entry\'s kind on the log path', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) + await drain() + const log = dispatches.find(d => d.state === TestFrameworkState.LOG)! + expect((log.data.test_logs as Record[])[0].kind).toBe('TEST_SCREENSHOT') + }) + + it('is not idle until queued events are dispatched', async () => { + framework.onReporterTestStart(testStats() as any, context()) + expect(WdioJasmineTestFramework.isIdle()).toBe(false) + await drain() + expect(WdioJasmineTestFramework.isIdle()).toBe(true) + }) + + it('logs and continues when an observer throws', async () => { + vi.mocked(framework.runHooks).mockRejectedValueOnce(new Error('observer blew up')) + const stats = testStats() + framework.onReporterTestStart(stats as any, context()) + framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + await drain() + expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) + expect(WdioJasmineTestFramework.isIdle()).toBe(true) + }) +}) diff --git a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts new file mode 100644 index 00000000..537c8b55 --- /dev/null +++ b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts @@ -0,0 +1,188 @@ +import path from 'node:path' +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +import TestReporter from '../src/reporter.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' +import * as utils from '../src/util.js' +import * as bstackLogger from '../src/bstackLogger.js' + +vi.mock('uuid', () => ({ v4: () => 'legacy-uuid' })) +vi.mock('@wdio/reporter', () => import(path.join(process.cwd(), '__mocks__', '@wdio/reporter'))) +vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +const runnerConfig = { + type: 'runner', + cid: '0-0', + capabilities: { browserName: 'chrome', browserVersion: '151' }, + config: { framework: 'jasmine', hostname: 'hub.browserstack.com' }, + specs: ['/work/test/p2/nested.spec.js'], + sessionId: 'sessionId' +} + +const suite = { title: 'Nested outer', file: '/work/test/p2/nested.spec.js' } +const testStats = () => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + _duration: 0, + retries: 0, + state: 'pending', +}) +const hookStats = () => ({ type: 'hook', uid: 'h', title: '"before all" hook', start: new Date(), _duration: 0 }) +const logEntry = () => ({ timestamp: new Date().toISOString(), level: 'INFO', message: 'console line', kind: 'TEST_LOG' as const, http_response: {} }) + +describe('reporter jasmine arm', () => { + let reporter: TestReporter + let listener: Record> + let framework: WdioJasmineTestFramework + + const setCli = (running: boolean, testFramework: unknown) => { + vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => running, + getTestFramework: () => testFramework, + } as unknown as BrowserstackCLI) + } + + beforeEach(async () => { + vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') + vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') + for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { + vi.spyOn(framework, m) + } + vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') + vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterLog).mockReturnValue(undefined) + + reporter = new TestReporter({}) + await reporter.onRunnerStart(runnerConfig as any) + reporter.onSuiteStart(suite as any) + listener = { + testStarted: vi.spyOn(reporter['listener'], 'testStarted').mockImplementation(() => {}), + testFinished: vi.spyOn(reporter['listener'], 'testFinished').mockImplementation(() => {}), + hookStarted: vi.spyOn(reporter['listener'], 'hookStarted').mockImplementation(() => {}), + hookFinished: vi.spyOn(reporter['listener'], 'hookFinished').mockImplementation(() => {}), + logCreated: vi.spyOn(reporter['listener'], 'logCreated').mockImplementation(() => {}), + } + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + describe('on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('feeds every test and hook event to the framework and enqueues nothing on the legacy Listener', async () => { + const stats = testStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + await reporter.appendTestItemLog(logEntry()) + + expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterLog).toHaveBeenCalledTimes(1) + for (const spy of Object.values(listener)) { + expect(spy).not.toHaveBeenCalled() + } + }) + + it('passes the suite stack and suite file', async () => { + await reporter.onTestStart(testStats() as any) + expect(framework.onReporterTestStart).toHaveBeenCalledWith( + expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), + { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } + ) + }) + + it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { + await reporter.onTestStart(testStats() as any) + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) + }) + + it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { + const stats = testStats() as Record + await reporter.onTestEnd(stats as any) + expect(stats.end).toBeInstanceOf(Date) + const hook = hookStats() as Record + await reporter.onHookEnd(hook as any) + expect(hook.state).toBe('passed') + }) + + it('drops ', async () => { + await reporter.onTestStart({ ...testStats(), fullTitle: '' } as any) + await reporter.onTestEnd({ ...testStats(), fullTitle: '' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterTestEnd).not.toHaveBeenCalled() + }) + + it('sends nothing when Test Observability is opted out', async () => { + reporter['_observability'] = false + await reporter.onTestStart(testStats() as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + }) + + it('never falls back to the legacy Listener when the tracker is missing', async () => { + setCli(true, null) + await reporter.onTestStart(testStats() as any) + await reporter.appendTestItemLog(logEntry()) + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.logCreated).not.toHaveBeenCalled() + }) + + it('reports unsynchronised while the framework has queued events', () => { + const spy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) + expect(reporter.isSynchronised).toBe(false) + spy.mockReturnValue(true) + expect(reporter.isSynchronised).toBe(true) + }) + }) + + describe('on the legacy flow', () => { + beforeEach(() => setCli(false, null)) + + it('enqueues on the legacy Listener and never touches the framework', async () => { + const stats = testStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + reporter['_currentHook'] = {} + await reporter.appendTestItemLog(logEntry()) + + expect(listener.hookStarted).toHaveBeenCalledTimes(1) + expect(listener.hookFinished).toHaveBeenCalledTimes(1) + expect(listener.testStarted).toHaveBeenCalledTimes(1) + expect(listener.testFinished).toHaveBeenCalledTimes(1) + expect(listener.logCreated).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterLog).not.toHaveBeenCalled() + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'legacy-uuid' }) + }) + }) + + describe('mocha on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('leaves the mocha arm unchanged: no start/end publishing from the reporter', async () => { + reporter['_config']!.framework = 'mocha' + await reporter.onTestStart(testStats() as any) + await reporter.onTestEnd({ ...testStats(), state: 'passed' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.testFinished).not.toHaveBeenCalled() + }) + }) +}) From 362ee0e34e3c5fa4b6a6cbe3da954f5c16c85ba0 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 13:43:42 +0530 Subject: [PATCH 03/15] fix(browserstack-service): route jasmine reporter events to TestHub only (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 113 +++++++++++++++--- .../src/cli/modules/testHubModule.ts | 7 ++ packages/browserstack-service/src/reporter.ts | 10 +- .../modules/testHubModule.skipMarker.test.ts | 57 +++++++++ .../cli/wdioJasmineTestFramework.test.ts | 109 +++++++++++++---- 5 files changed, 255 insertions(+), 41 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 45dde1af..fe717388 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -15,6 +15,7 @@ import { BStackLogger as logger } from '../cliLogger.js' import { getHookType, getTestTags, removeAnsiColors } from '../../util.js' import { shouldProcessEventForTesthub } from '../../testHub/utils.js' import { TEST_ANALYTICS_ID } from '../../constants.js' +import type TestHubModule from '../modules/testHubModule.js' /** What the reporter knows at each event and the framework cannot see: its suite stack and suite file. */ export interface JasmineSuiteContext { @@ -34,9 +35,12 @@ interface FailureFields { * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), * hook taxonomy and skip/exclude states differ from mocha's. * - * Test and hook states come from the WDIO reporter (`reporter.ts`), the only jasmine source that - * sees pending, focused-out and beforeAll-failed specs. Reporter hooks are not awaited by WDIO, so - * every reporter event is applied and dispatched through one ordered queue. + * Two event sources, as on the legacy flow: + * - the WDIO reporter (`reporter.ts`) feeds Test Observability only — it is the one jasmine source that + * sees pending, focused-out and beforeAll-failed specs. Its hooks are not awaited by WDIO, so its events + * are applied through one ordered queue and sent straight to TestHubModule, never to the other modules; + * - the service's awaited beforeTest/afterTest drive the product modules (a11y, Automate, Percy) through + * the observers, for executed specs only, with TestHub skipped. */ export default class WdioJasmineTestFramework extends TestFramework { // Same data keys WdioMochaTestFramework uses: TestHubModule's LOG arm reads them through @@ -56,6 +60,11 @@ export default class WdioJasmineTestFramework extends TestFramework { static #pendingEvents = 0 #specInstances = new Map() + // Open specs by fullName: how the service's hooks find the instance the reporter minted. + #specsByFullName = new Map() + #serviceOnly = new Set() + #suiteTitles = new Map() + #testHub: TestHubModule | null = null // beforeAll/afterAll never overlap, so a hook's uid (its title) is unique while it is open. #hookInstances = new Map() #openHook: TestFrameworkInstance | null = null @@ -72,18 +81,47 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#pendingEvents === 0 } + setTestHubModule(testHub: TestHubModule | null | undefined) { + this.#testHub = testHub ?? null + } + /** - * Service hooks (beforeTest/afterTest and friends) also call this. Their TEST/INIT_TEST/LOG_REPORT - * states would duplicate what the reporter already dispatched, so they are absorbed; waiting for the - * queue first leaves the tracked instance pointing at the spec the service is running. + * The service's hooks call this. INIT_TEST pins the spec's instance and uuid; TEST PRE/POST reach the + * product modules. LOG_REPORT carries nothing jasmine needs: the result comes from the reporter. */ async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { if (testFrameworkState === TestFrameworkState.LOG) { this.onReporterLog(args.logEntry as Record) return } - await this.#queue - logger.debug(`WdioJasmineTestFramework: absorbed service event ${testFrameworkState}/${hookState}`) + try { + await this.#queue + if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { + return + } + const test = (args.test ?? {}) as { fullName?: string, description?: string, file?: string } + const instance = this.#serviceInstance(test) + TestFramework.setTrackedInstance(instance.getContext(), instance) + process.env[TEST_ANALYTICS_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) + if (testFrameworkState === TestFrameworkState.INIT_TEST) { + return + } + + const moduleArgs: Record = { ...args } + if (hookState === HookState.PRE) { + this.#suiteTitles.set(instance, args.suiteTitle) + } else { + // afterTest hands over the raw `Jasmine__TopLevel__Suite`; the modules need the describe chain beforeTest derived + moduleArgs.suiteTitle = this.#suiteTitles.get(instance) ?? args.suiteTitle + this.#suiteTitles.delete(instance) + if (this.#serviceOnly.delete(instance) && test.fullName && this.#specsByFullName.get(test.fullName) === instance) { + this.#specsByFullName.delete(test.fullName) + } + } + await this.#toModules(instance, testFrameworkState, hookState, moduleArgs) + } catch (error) { + logger.error(`WdioJasmineTestFramework: service event ${testFrameworkState}/${hookState} failed: ${util.format(error)}`) + } } onReporterTestStart(testStats: TestStats, context: JasmineSuiteContext) { @@ -106,14 +144,14 @@ export default class WdioJasmineTestFramework extends TestFramework { [WdioJasmineTestFramework.KEY_TEST_RETRIES]: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, }) this.#specInstances.set(testStats.uid, instance) + this.#specsByFullName.set(fullTitle, instance) // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. TestFramework.setTrackedInstance(instance.getContext(), instance) const args = { test: this.#specArg(testStats, context), suiteTitle: this.#suiteTitle(testStats) } this.#enqueue('TEST/PRE', async () => { this.#lastSpec = instance - process.env[TEST_ANALYTICS_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) - await this.#emit(instance, TestFrameworkState.TEST, HookState.PRE, args) + await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.PRE, args) }) return TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string } catch (error) { @@ -129,6 +167,9 @@ export default class WdioJasmineTestFramework extends TestFramework { return } this.#specInstances.delete(testStats.uid) + if (this.#specsByFullName.get(testStats.fullTitle) === instance) { + this.#specsByFullName.delete(testStats.fullTitle) + } const state = testStats.state const error = testStats.error @@ -162,7 +203,7 @@ export default class WdioJasmineTestFramework extends TestFramework { const args = { test: this.#specArg(testStats, context), result: testResult, suiteTitle: this.#suiteTitle(testStats) } this.#enqueue('TEST/POST', async () => { instance.updateMultipleEntries(entries) - await this.#emit(instance, TestFrameworkState.TEST, HookState.POST, args) + await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.POST, args) }) } catch (error) { logger.error(`WdioJasmineTestFramework: test end failed: ${util.format(error)}`) @@ -204,7 +245,7 @@ export default class WdioJasmineTestFramework extends TestFramework { hooksStarted.set(key, [hook]) instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_STARTED, key) if (emitHook) { - await this.#emit(instance, hookFrameworkState, HookState.PRE, { hook: hookStats }) + await this.#toTestHub(instance, hookFrameworkState, HookState.PRE, { hook: hookStats }) } }) } catch (error) { @@ -258,7 +299,7 @@ export default class WdioJasmineTestFramework extends TestFramework { hooksFinished.set(key, [hook]) instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_FINISHED, key) if (emitHook) { - await this.#emit(instance, hookFrameworkState, HookState.POST, { hook: hookStats, result: { passed: result === 'passed', error } }) + await this.#toTestHub(instance, hookFrameworkState, HookState.POST, { hook: hookStats, result: { passed: result === 'passed', error } }) } }) } catch (error) { @@ -286,9 +327,11 @@ export default class WdioJasmineTestFramework extends TestFramework { if (instance === this.#openHook) { logRecord[TestFrameworkConstants.KEY_HOOK_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) } - const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] - entries.push(logRecord) - await this.#emit(instance, TestFrameworkState.LOG, HookState.POST, { logEntry }) + // Sent in the instance's own state, not LOG: the binary keys an entry `hook_run_uuid` only when + // its state is a hook state, so an open all-hook's logs land on the hook. + if (this.#testHub) { + await this.#testHub.sendLogCreatedEvent({ instance, logEntries: [logRecord] }) + } }) } catch (error) { logger.error(`WdioJasmineTestFramework: log failed: ${util.format(error)}`) @@ -315,13 +358,47 @@ export default class WdioJasmineTestFramework extends TestFramework { return instance } - async #emit(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { - TestFramework.setTrackedInstance(instance.getContext(), instance) + async #toTestHub(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + const testHub = this.#testHub + if (!testHub) { + logger.debug(`WdioJasmineTestFramework: TestHub module not loaded; ${testFrameworkState}/${hookState} not reported`) + return + } this.updateInstanceState(instance, testFrameworkState, hookState) args.instance = instance + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + testHub.onBeforeTest(args) + } + await testHub.sendTestFrameworkEvent(args) + } + + async #toModules(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + args.instance = instance + args.skipTestHub = true await this.runHooks(instance, testFrameworkState, hookState, args) } + /** The reporter's instance for this spec; minted here only when the reporter is not feeding (TO opted out). */ + #serviceInstance(test: { fullName?: string, description?: string, file?: string }) { + const existing = test.fullName ? this.#specsByFullName.get(test.fullName) : undefined + if (existing) { + return existing + } + const instance = this.#createInstance() + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + [TestFrameworkConstants.KEY_TEST_ID]: test.fullName, + [TestFrameworkConstants.KEY_TEST_NAME]: test.description, + ...this.#filePaths(test.file), + }) + this.#serviceOnly.add(instance) + if (test.fullName) { + this.#specsByFullName.set(test.fullName, instance) + } + return instance + } + #enqueue(label: string, step: () => Promise) { WdioJasmineTestFramework.#pendingEvents++ this.#queue = this.#queue.then(step).catch((error) => { diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index 5026bdaa..1830be1e 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -82,6 +82,10 @@ export default class TestHubModule extends BaseModule { } onBeforeTest(args: Record) { + // Jasmine reports to TestHub from its reporter feed; its module-only dispatches carry this marker + if (args.skipTestHub === true) { + return + } this.logger.debug('onBeforeTest: Called after test hook from cli configured module!!!') const autoInstance = AutomationFramework.getTrackedInstance() as AutomationFrameworkInstance const instances = [autoInstance] @@ -90,6 +94,9 @@ export default class TestHubModule extends BaseModule { } onAllTestEvents(args: Record) { + if (args.skipTestHub === true) { + return + } this.logger.debug('onAllTestEvents: Called after all test events from cli configured module!!!') const instance = args.instance as TestFrameworkInstance const testState = instance.getCurrentTestState() diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index 3e684124..b593e5c8 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -7,6 +7,7 @@ import { BrowserstackCLI } from './cli/index.js' import { reportSkippedTest, resolveSpecFile } from './cli/skipReporter.js' import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' import type { JasmineSuiteContext } from './cli/frameworks/wdioJasmineTestFramework.js' +import type TestHubModule from './cli/modules/testHubModule.js' import * as url from 'node:url' import { v4 as uuidv4 } from 'uuid' @@ -154,8 +155,13 @@ class _TestReporter extends WDIOReporter { } cliJasmineFramework() { - const framework = BrowserstackCLI.getInstance().getTestFramework() - return framework instanceof WdioJasmineTestFramework ? framework : null + const cli = BrowserstackCLI.getInstance() + const framework = cli.getTestFramework() + if (!(framework instanceof WdioJasmineTestFramework)) { + return null + } + framework.setTestHubModule(cli.modules?.TestHubModule as TestHubModule | undefined) + return framework } jasmineSuiteContext(): JasmineSuiteContext { diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts new file mode 100644 index 00000000..3dbba978 --- /dev/null +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' + +import * as bstackLogger from '../../../src/bstackLogger.js' +import TestHubModule from '../../../src/cli/modules/testHubModule.js' +import TestFrameworkInstance from '../../../src/cli/instances/testFrameworkInstance.js' +import TrackedInstance from '../../../src/cli/instances/trackedInstance.js' +import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../../src/cli/states/hookState.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +describe('TestHubModule skipTestHub marker', () => { + let module: TestHubModule + let sendTestFrameworkEvent: ReturnType + let sendTestSessionEvent: ReturnType + + const instanceIn = (state: State, hook: State, framework = 'WebdriverIO-cucumber') => { + const instance = new TestFrameworkInstance(TrackedInstance.createContext('t'), [framework], { [framework]: '9' }, state, hook) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: framework, + [TestFrameworkConstants.KEY_TEST_UUID]: 'uuid-1', + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: 'now', + }) + return instance + } + + beforeEach(() => { + module = new TestHubModule({}) + sendTestFrameworkEvent = vi.spyOn(module, 'sendTestFrameworkEvent').mockResolvedValue(true) + sendTestSessionEvent = vi.spyOn(module, 'sendTestSessionEvent').mockResolvedValue(undefined) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('handles test and hook events that carry no marker, as before', () => { + module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.BEFORE_ALL, HookState.POST) }) + expect(sendTestSessionEvent).toHaveBeenCalledTimes(1) + expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(2) + }) + + it('ignores events marked skipTestHub', () => { + module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: true }) + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.POST), skipTestHub: true }) + expect(sendTestSessionEvent).not.toHaveBeenCalled() + expect(sendTestFrameworkEvent).not.toHaveBeenCalled() + }) + + it('only honours the literal true', () => { + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: 'true' }) + expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index 8eec475f..2a459047 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -7,6 +7,7 @@ import TestFramework from '../../src/cli/frameworks/testFramework.js' import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' import { HookState } from '../../src/cli/states/hookState.js' import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' +import TestHubModule from '../../src/cli/modules/testHubModule.js' vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) @@ -50,16 +51,37 @@ const context = (scopes = ['Nested outer']) => ({ scopes, suiteFile: SUITE_FILE describe('WdioJasmineTestFramework', () => { let framework: WdioJasmineTestFramework + // what reached TestHub (reporter path) and what reached the module observers (service path) let dispatches: Dispatch[] + let moduleDispatches: Dispatch[] + let logSends: Array<{ state: string, entries: Record[], data: Record }> + let sessionEvents: number + let testHub: Record> - const drain = () => framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, {}) + const drain = () => framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, {}) beforeEach(() => { process.env.BROWSERSTACK_OBSERVABILITY = 'true' framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') dispatches = [] + moduleDispatches = [] + logSends = [] + sessionEvents = 0 + testHub = { + onBeforeTest: vi.fn(() => { sessionEvents++ }), + sendTestFrameworkEvent: vi.fn(async (args: Record) => { + const instance = args.instance as TestFrameworkInstance + dispatches.push({ state: instance.getCurrentTestState(), hook: instance.getCurrentHookState(), data: snapshot(instance), args }) + return true + }), + sendLogCreatedEvent: vi.fn(async (args: Record) => { + const instance = args.instance as TestFrameworkInstance + logSends.push({ state: instance.getCurrentTestState().toString(), entries: args.logEntries as Record[], data: snapshot(instance) }) + }), + } + framework.setTestHubModule(testHub as unknown as TestHubModule) vi.spyOn(framework, 'runHooks').mockImplementation(async (instance, state, hook, args) => { - dispatches.push({ state, hook, data: snapshot(instance), args: args as Record }) + moduleDispatches.push({ state, hook, data: snapshot(instance), args: args as Record }) }) }) @@ -231,24 +253,64 @@ describe('WdioJasmineTestFramework', () => { framework.onReporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) await drain() expect(dispatches.map(d => d.state)).toEqual([TestFrameworkState.TEST]) + expect(logSends).toHaveLength(0) } finally { delete process.env.BROWSERSTACK_ACCESSIBILITY } }) - it('absorbs service-hook states instead of re-dispatching them, leaving the spec tracked', async () => { + it('sends reporter events to TestHub only, never through the module observers', async () => { + const stats = testStats() + framework.onReporterTestStart(stats as any, context()) + framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + await drain() + expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(2) + expect(sessionEvents).toBe(1) + expect(moduleDispatches).toHaveLength(0) + }) + + it('drives the modules from the service hooks on the reporter\'s instance, with TestHub skipped', async () => { + const spec = { description: 'outer passing test', fullName: 'Nested outer outer passing test' } framework.onReporterTestStart(testStats() as any, context()) - await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: {} }) - const tracked = TestFramework.getTrackedInstance() - expect(TestFramework.getState(tracked, 'test_uuid')).toBe(dispatches[0].data.test_uuid) + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) + const uuid = dispatches[0].data.test_uuid + expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) + expect(process.env.TEST_ANALYTICS_ID).toBe(uuid) + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: spec, suiteTitle: 'Nested outer' }) + await framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test: spec, result: {} }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: spec, result: { passed: true }, suiteTitle: 'Jasmine__TopLevel__Suite' }) + + expect(moduleDispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) + for (const d of moduleDispatches) { + expect(d.args.skipTestHub).toBe(true) + expect(d.data.test_uuid).toBe(uuid) + } + expect(moduleDispatches[1].args.suiteTitle).toBe('Nested outer') + expect(moduleDispatches[1].args.result).toEqual({ passed: true }) + expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(1) + }) - await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: {} }) - await framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test: {}, result: {} }) - await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: {}, result: {} }) - expect(dispatches).toHaveLength(1) + it('never shows a spec the reporter saw but the service did not (pending, excluded, beforeAll-failed) to the modules', async () => { + framework.onReporterTestStart(testStats({ uid: 'x' }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'x', state: 'skipped', end: new Date() }) as any, context()) + await drain() + expect(dispatches).toHaveLength(2) + expect(moduleDispatches).toHaveLength(0) + }) + + it('still drives the modules when Test Observability is opted out and the reporter feeds nothing', async () => { + const spec = { description: 'a', fullName: 'Suite a', file: SUITE_FILE } + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: spec, suiteTitle: 'Suite' }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: spec, result: { passed: true } }) + expect(moduleDispatches).toHaveLength(2) + expect(moduleDispatches[0].data.test_uuid).toBe(moduleDispatches[1].data.test_uuid) + expect(process.env.TEST_ANALYTICS_ID).toBe(moduleDispatches[0].data.test_uuid) + expect(testHub.sendTestFrameworkEvent).not.toHaveBeenCalled() }) - it('attributes logs to an open all-hook, else to the last-started spec', async () => { + it('sends an open all-hook\'s logs in the hook state, else the last-started spec\'s in the test state', async () => { framework.onReporterTestStart(testStats() as any, context()) framework.onReporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) const after = hookStats('"after all" hook') @@ -258,22 +320,20 @@ describe('WdioJasmineTestFramework', () => { framework.onReporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) await drain() - const logs = dispatches.filter(d => d.state === TestFrameworkState.LOG) - expect(logs).toHaveLength(3) + expect(logSends.map(l => l.state)).toEqual(['TestFrameworkState.TEST', 'TestFrameworkState.AFTER_ALL', 'TestFrameworkState.TEST']) const specUuid = dispatches[0].data.test_uuid - expect(logs[0].data.test_uuid).toBe(specUuid) - expect((logs[0].data.test_logs as Record[]).at(-1)).not.toHaveProperty('hook_id') - const hookLog = (logs[1].data.test_logs as Record[]).at(-1)! - expect(hookLog.hook_id).toBe((dispatches.find(d => d.state === TestFrameworkState.AFTER_ALL)!.data.test_hooks_started as Record[]>).AFTER_ALL[0].hook_id) - expect(logs[2].data.test_uuid).toBe(specUuid) + expect(logSends[0].data.test_uuid).toBe(specUuid) + expect(logSends[0].entries[0]).not.toHaveProperty('hook_id') + const hookId = (dispatches.find(d => d.state === TestFrameworkState.AFTER_ALL)!.data.test_hooks_started as Record[]>).AFTER_ALL[0].hook_id + expect(logSends[1].entries[0].hook_id).toBe(hookId) + expect(logSends[2].data.test_uuid).toBe(specUuid) }) it('keeps a screenshot entry\'s kind on the log path', async () => { framework.onReporterTestStart(testStats() as any, context()) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) await drain() - const log = dispatches.find(d => d.state === TestFrameworkState.LOG)! - expect((log.data.test_logs as Record[])[0].kind).toBe('TEST_SCREENSHOT') + expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') }) it('is not idle until queued events are dispatched', async () => { @@ -284,7 +344,7 @@ describe('WdioJasmineTestFramework', () => { }) it('logs and continues when an observer throws', async () => { - vi.mocked(framework.runHooks).mockRejectedValueOnce(new Error('observer blew up')) + testHub.sendTestFrameworkEvent.mockRejectedValueOnce(new Error('send blew up')) const stats = testStats() framework.onReporterTestStart(stats as any, context()) framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) @@ -292,4 +352,11 @@ describe('WdioJasmineTestFramework', () => { expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) expect(WdioJasmineTestFramework.isIdle()).toBe(true) }) + + // The class calls these TestHubModule methods directly; renaming or removing one must fail here. + it('relies on TestHubModule handlers that exist', () => { + for (const method of ['onBeforeTest', 'sendTestFrameworkEvent', 'sendLogCreatedEvent']) { + expect(typeof (TestHubModule.prototype as unknown as Record)[method]).toBe('function') + } + }) }) From 1a7d7c73286c1c12898dd5943a02a72b27c0489d Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 14:50:51 +0530 Subject: [PATCH 04/15] feat(browserstack-service): jasmine shared dispatch gates and HTTP logs on CLI (SDK-7772) - service hooks classify jasmine hooks by the WDIO hookName instead of the mocha title; the mocha skip cascade stays mocha-only - beforeTest annotates each jasmine spec with `Test: ` on the CLI flow - jasmine registers the command listener on the CLI flow; browserCommand sends HTTP command logs over gRPC (LogEntry kind HTTP) to the spec it names, and jasmine screenshots name their spec the same way - the jasmine class sends a session event for every beforeAll/afterAll start so hook runs carry their platform and Automate session, as on the legacy flow Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 27 +- .../src/insights-handler.ts | 28 +- packages/browserstack-service/src/service.ts | 46 +++- .../cli/wdioJasmineTestFramework.test.ts | 45 +++- .../tests/service.jasmineCliGates.test.ts | 254 ++++++++++++++++++ 5 files changed, 378 insertions(+), 22 deletions(-) create mode 100644 packages/browserstack-service/tests/service.jasmineCliGates.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index fe717388..649e8366 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -58,10 +58,18 @@ export default class WdioJasmineTestFramework extends TestFramework { static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' static #pendingEvents = 0 + static #hookTypes = new Map([ + ['beforeAll', 'BEFORE_ALL'], + ['afterAll', 'AFTER_ALL'], + ['beforeEach', 'BEFORE_EACH'], + ['afterEach', 'AFTER_EACH'], + ]) #specInstances = new Map() // Open specs by fullName: how the service's hooks find the instance the reporter minted. #specsByFullName = new Map() + // Every spec of this worker by uuid: legacy attaches HTTP logs and screenshots to the last started spec, even after it ended. + #specsByUuid = new Map() #serviceOnly = new Set() #suiteTitles = new Map() #testHub: TestHubModule | null = null @@ -81,6 +89,11 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#pendingEvents === 0 } + /** The WDIO `hookName` a jasmine hook runs under, as the hook-type key `getHookType` returns for mocha titles. */ + static hookTypeFromName(hookName: string | undefined) { + return WdioJasmineTestFramework.#hookTypes.get(hookName ?? '') ?? 'unknown' + } + setTestHubModule(testHub: TestHubModule | null | undefined) { this.#testHub = testHub ?? null } @@ -145,6 +158,7 @@ export default class WdioJasmineTestFramework extends TestFramework { }) this.#specInstances.set(testStats.uid, instance) this.#specsByFullName.set(fullTitle, instance) + this.#specsByUuid.set(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string, instance) // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. TestFramework.setTrackedInstance(instance.getContext(), instance) @@ -236,6 +250,8 @@ export default class WdioJasmineTestFramework extends TestFramework { ...this.#filePaths(context.suiteFile), } instance.updateData(TestFrameworkConstants.KEY_HOOK_ID, hookId) + // The session event is keyed by KEY_TEST_UUID: it links the hook run to its Automate session and platform + instance.updateData(TestFrameworkConstants.KEY_TEST_UUID, hookId) this.#hookInstances.set(hookStats.uid, instance) const emitHook = shouldProcessEventForTesthub('HookRunStarted') @@ -307,14 +323,18 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - /** Console logs and screenshots: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. */ + /** + * Console logs: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. + * HTTP command logs and screenshots name their spec (`test_run_uuid`), as legacy did; unknown uuids are dropped. + */ onReporterLog(logEntry: Record | undefined) { try { if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { return } + const targetUuid = logEntry.test_run_uuid as string | undefined this.#enqueue('LOG/POST', async () => { - const instance = this.#openHook ?? this.#lastSpec + const instance = targetUuid ? this.#specsByUuid.get(targetUuid) : (this.#openHook ?? this.#lastSpec) if (!instance) { return } @@ -366,7 +386,8 @@ export default class WdioJasmineTestFramework extends TestFramework { } this.updateInstanceState(instance, testFrameworkState, hookState) args.instance = instance - if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + // Legacy sent a run's platform and session with its start event, hooks included (parity #19, #23) + if (hookState === HookState.PRE) { testHub.onBeforeTest(args) } await testHub.sendTestFrameworkEvent(args) diff --git a/packages/browserstack-service/src/insights-handler.ts b/packages/browserstack-service/src/insights-handler.ts index 3542882b..1dc36338 100644 --- a/packages/browserstack-service/src/insights-handler.ts +++ b/packages/browserstack-service/src/insights-handler.ts @@ -812,7 +812,8 @@ class _InsightsHandler { logEntry: { kind: TestFrameworkConstants.KIND_SCREENSHOT, message: result.value, - timestamp: new Date().toISOString() + timestamp: new Date().toISOString(), + ...(this._framework === 'jasmine' ? { test_run_uuid: testMeta.uuid } : {}) } }) : this.listener.onScreenshot([{ @@ -829,16 +830,29 @@ class _InsightsHandler { } // log http request + const httpResponse = { + path: requestData.endpoint, + method: requestData.method, + body, + response: result + } + // Only jasmine registers `command` on the CLI flow, and the legacy listener is inert there + if (BrowserstackCLI.getInstance().isRunning()) { + await BrowserstackCLI.getInstance().getTestFramework()?.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { + kind: 'HTTP', + message: JSON.stringify(httpResponse), + timestamp: new Date().toISOString(), + test_run_uuid: testMeta.uuid + } + }) + return + } this.listener.logCreated([{ test_run_uuid: testMeta.uuid, timestamp: new Date().toISOString(), kind: 'HTTP', - http_response: { - path: requestData.endpoint, - method: requestData.method, - body, - response: result - } + http_response: httpResponse }] ) } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index e262f5ca..104a1a5d 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -45,6 +45,7 @@ import { HookState } from './cli/states/hookState.js' import { AutomationFrameworkConstants } from './cli/frameworks/constants/automationFrameworkConstants.js' import TestFramework from './cli/frameworks/testFramework.js' import WdioCucumberTestFramework from './cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' import { TestFrameworkState } from './cli/states/testFrameworkState.js' import { TestFrameworkConstants } from './cli/frameworks/constants/testFrameworkConstants.js' import AccessibilityModule from './cli/modules/accessibilityModule.js' @@ -330,12 +331,22 @@ export default class BrowserstackService implements Services.ServiceInstance { * `browserCommand` is the only producer of TEST_SCREENSHOT logs — the binary * has no screenshot producer of its own — so the result event has to be * registered on this path too, or a screenshot taken mid-test never reaches - * Observability (SDK-4177). The `command` (beforeCommand) event is - * deliberately NOT registered: it only fills the map that browserCommand's - * HTTP-log half reads, and that half emits on the JS listener pipeline the - * binary owns here. Leaving it unregistered keeps the screenshot upload — - * which rides its own JWT-authenticated endpoint — as the single effect. + * Observability (SDK-4177). The `command` (beforeCommand) event only fills + * the map browserCommand's HTTP-log half reads, so it is registered for + * jasmine alone, whose HTTP command logs go over gRPC; mocha and cucumber + * never reported them on this path. */ + if (this._config.framework === 'jasmine') { + this._browser.on('command', (command) => { + if (shouldProcessEventForTesthub('')) { + this._insightsHandler?.browserCommand( + 'client:beforeCommand', + Object.assign(command, { sessionId }), + this._currentTest + ) + } + }) + } this._browser.on('result', (result) => { if (shouldProcessEventForTesthub('')) { this._insightsHandler?.browserCommand( @@ -450,7 +461,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeHook' }) - async beforeHook (test: Frameworks.Test|CucumberHook, context: unknown) { + async beforeHook (test: Frameworks.Test|CucumberHook, context: unknown, hookName?: string) { if (this._config.framework !== 'cucumber') { this._currentTest = test as Frameworks.Test // not update currentTest when this is called for cucumber step } @@ -481,7 +492,7 @@ export default class BrowserstackService implements Services.ServiceInstance { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) } } else if (framework) { - const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] + const hookFrameworkState = TestFrameworkState[this._cliHookType(test as Frameworks.Test, hookName) as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) } @@ -496,7 +507,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterHook' }) - async afterHook(test: Frameworks.Test | CucumberHook, context: unknown, result: Frameworks.TestResult) { + async afterHook(test: Frameworks.Test | CucumberHook, context: unknown, result: Frameworks.TestResult, hookName?: string) { // The Mocha hook window is closed — clear the tracker (see beforeHook). if (this._config.framework === 'mocha') { setCurrentMochaHookWindow(null) @@ -535,16 +546,16 @@ export default class BrowserstackService implements Services.ServiceInstance { return } if (framework) { - const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] + const hookType = this._cliHookType(test as Frameworks.Test, hookName) + const hookFrameworkState = TestFrameworkState[hookType as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result }) } // a failed (or skipping) before/each hook silently drops the suite's remaining // tests in mocha — report them as skipped so they surface on the dashboard and // attribute their Automate session (port of the legacy insights-handler cascade) - const hookType = getHookType((test as Frameworks.Test).title) const suite = (test as Frameworks.Test).ctx?.test?.parent - if (result && !result.passed && ['BEFORE_ALL', 'BEFORE_EACH', 'AFTER_EACH'].includes(hookType) && suite) { + if (this._config.framework === 'mocha' && result && !result.passed && ['BEFORE_ALL', 'BEFORE_EACH', 'AFTER_EACH'].includes(hookType) && suite) { await reportSuiteSkipped(framework, suite) } } @@ -555,6 +566,16 @@ export default class BrowserstackService implements Services.ServiceInstance { await this._accessibilityHandler?.afterHook() } + /** + * Jasmine's hook argument is a copy of the last started spec, never the hook itself, so only + * the `hookName` WDIO passes says which hook is running. + */ + private _cliHookType(test: Frameworks.Test, hookName?: string) { + return this._config.framework === 'jasmine' + ? WdioJasmineTestFramework.hookTypeFromName(hookName) + : getHookType(test.title) + } + /** * Cucumber abandons the whole feature when a `BeforeAll` throws, so every scenario in it * (Rule-nested included) is reported SKIPPED rather than vanishing. Ports @@ -623,6 +644,9 @@ export default class BrowserstackService implements Services.ServiceInstance { // skip reporter must never re-report it from onTestSkip markTestStarted(getUniqueIdentifier(test, this._config.framework)) this._insightsHandler?.setTestData(test, uuid) + if (this._config.framework === 'jasmine') { + await this._setAnnotation(`Test: ${test.fullName ?? test.title}`) + } await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle }) return } diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index 2a459047..cfce5f23 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -214,7 +214,11 @@ describe('WdioJasmineTestFramework', () => { expect(finished).toMatchObject({ hook_result: 'passed', event_ended_at: '2026-09-25T15:34:12.257Z', hook_duration: 4087 }) expect(finished.hook_id).toBe(started.hook_id) expect('hook_failure' in finished).toBe(false) - expect('test_uuid' in dispatches[0].data).toBe(false) + // the session event is keyed by the hook's own uuid, so the hook run gets its platform and session (#23); + // the binary builds the HookRun from hook_id and never links it to a test (#25) + expect(dispatches[0].data.test_uuid).toBe(started.hook_id) + expect(testHub.onBeforeTest).toHaveBeenCalledTimes(2) + expect(testHub.onBeforeTest.mock.calls.map(([args]) => (args as Record).instance)).toEqual([dispatches[0].args.instance, dispatches[2].args.instance]) const afterStarted = (dispatches[2].data.test_hooks_started as Record[]>).AFTER_ALL[0] expect(afterStarted.hook_identifier).toBe('"after all" hook for Nested middle') @@ -336,6 +340,45 @@ describe('WdioJasmineTestFramework', () => { expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') }) + it('sends an HTTP command log to the spec it names, even inside an all-hook or after the spec ended', async () => { + const first = testStats() + const uuid = framework.onReporterTestStart(first as any, context()) + framework.onReporterTestEnd({ ...first, state: 'passed', end: new Date() } as any, context()) + const after = hookStats('"after all" hook') + framework.onReporterHookStart(after as any, context()) + const message = JSON.stringify({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message, timestamp: 't', test_run_uuid: uuid } }) + await drain() + + expect(logSends).toHaveLength(1) + expect(logSends[0].state).toBe('TestFrameworkState.TEST') + expect(logSends[0].data.test_uuid).toBe(uuid) + expect(logSends[0].entries[0]).toMatchObject({ kind: 'HTTP', timestamp: 't' }) + expect(logSends[0].entries[0]).not.toHaveProperty('hook_id') + expect(Buffer.from(logSends[0].entries[0].message as Uint8Array).toString()).toBe(message) + }) + + it('drops a named-spec log whose uuid this worker never minted', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message: '{}', timestamp: 't', test_run_uuid: 'not-ours' } }) + await drain() + expect(logSends).toHaveLength(0) + }) + + it('maps the WDIO hookName to the hook-type key, and nothing else', () => { + expect(['beforeAll', 'afterAll', 'beforeEach', 'afterEach'].map(n => WdioJasmineTestFramework.hookTypeFromName(n))) + .toEqual(['BEFORE_ALL', 'AFTER_ALL', 'BEFORE_EACH', 'AFTER_EACH']) + expect(WdioJasmineTestFramework.hookTypeFromName(undefined)).toBe('unknown') + expect(WdioJasmineTestFramework.hookTypeFromName('constructor')).toBe('unknown') + }) + + it('ignores the service\'s hook events: hooks reach TestHub from the reporter only', async () => { + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { fullName: 'Nested outer outer passing test' } }) + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { test: {}, result: { passed: false } }) + expect(dispatches).toHaveLength(0) + expect(moduleDispatches).toHaveLength(0) + }) + it('is not idle until queued events are dispatched', async () => { framework.onReporterTestStart(testStats() as any, context()) expect(WdioJasmineTestFramework.isIdle()).toBe(false) diff --git a/packages/browserstack-service/tests/service.jasmineCliGates.test.ts b/packages/browserstack-service/tests/service.jasmineCliGates.test.ts new file mode 100644 index 00000000..3a891c8f --- /dev/null +++ b/packages/browserstack-service/tests/service.jasmineCliGates.test.ts @@ -0,0 +1,254 @@ +import path from 'node:path' + +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +import BrowserstackService from '../src/service.js' +import InsightsHandler from '../src/insights-handler.js' +import * as utils from '../src/util.js' +import * as skipReporter from '../src/cli/skipReporter.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import * as bstackLogger from '../src/bstackLogger.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' +import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' +import TestFramework from '../src/cli/frameworks/testFramework.js' + +vi.mock('fetch') +vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) +vi.mock('uuid', () => ({ v4: () => '123456789' })) +vi.mock('../src/data-store.js', () => ({ saveWorkerData: vi.fn() })) +vi.mock('../src/instrumentation/performance/performance-tester.js', () => ({ + default: { + start: vi.fn(), + end: vi.fn(), + startMonitoring: vi.fn(), + measureWrapper: vi.fn().mockImplementation((_name: string, fn: Function) => fn), + Measure: vi.fn().mockImplementation(() => (_target: any, _propertyKey: string, descriptor: PropertyDescriptor) => descriptor), + browser: undefined, + scenarioThatRan: [], + } +})) +vi.mock('../src/cli/index.js', () => ({ + BrowserstackCLI: { + getInstance: () => ({ + isRunning: () => false, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + }) + } +})) + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +// What WDIO hands a jasmine hook: a copy of the last started spec (no `title`), then the hookName. +const lastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any +const mochaHook = (title: string) => ({ title, ctx: { test: { parent: { title: 'suite', tests: [], suites: [] } } } }) as any + +let getInstanceSpy: ReturnType | undefined + +const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) +} + +const makeService = (framework: string) => new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + +afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined +}) + +describe('service hooks on the CLI flow (S4)', () => { + it('classifies jasmine hooks by hookName, without touching the missing title', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const getHookTypeSpy = vi.spyOn(utils, 'getHookType') + const service = makeService('jasmine') + + for (const hookName of ['beforeAll', 'beforeEach']) { + await service.beforeHook(lastSpec, {}, hookName) + } + for (const hookName of ['afterEach', 'afterAll']) { + await service.afterHook(lastSpec, {}, { passed: true } as any, hookName) + } + + expect(getHookTypeSpy).not.toHaveBeenCalled() + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.AFTER_EACH, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + getHookTypeSpy.mockRestore() + }) + + it('sends nothing for a jasmine hook without a hookName, and does not throw', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const service = makeService('jasmine') + + await expect(service.beforeHook({} as any, {})).resolves.toBeUndefined() + await expect(service.afterHook({} as any, {}, { passed: true } as any)).resolves.toBeUndefined() + expect(trackEvent).not.toHaveBeenCalled() + }) + + it('never runs the mocha skip cascade for a failed jasmine hook, but still records the hook failure', async () => { + cliWith({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('jasmine') + + await service.afterHook({ ...lastSpec, ctx: { test: { parent: {} } } }, {}, { passed: false, error: new Error('beforeAll failed') } as any, 'beforeAll') + + expect(cascade).not.toHaveBeenCalled() + expect(service['_hookFailReasons']).toEqual(['beforeAll failed']) + cascade.mockRestore() + }) + + it('keeps mocha on its title classification and skip cascade', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('mocha') + + await service.beforeHook(mochaHook('"before each" hook for "t"'), {}, 'beforeEach') + await service.afterHook(mochaHook('"before all" hook for "t"'), {}, { passed: false, error: new Error('x') } as any, 'afterAll') + + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + ]) + expect(cascade).toHaveBeenCalledTimes(1) + cascade.mockRestore() + }) +}) + +describe('service.beforeTest on the CLI flow (S5)', () => { + const runBeforeTest = async (framework: string, test: Record) => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const service = makeService(framework) + const annotate = vi.spyOn(service as any, '_setAnnotation').mockResolvedValue(undefined) + const getState = vi.spyOn(TestFramework, 'getState').mockReturnValue('spec-uuid') + await service.beforeTest(test as any) + getState.mockRestore() + return { annotate, trackEvent } + } + + it('annotates each jasmine spec with its full name, before the modules see TEST/PRE', async () => { + const { annotate, trackEvent } = await runBeforeTest('jasmine', lastSpec) + expect(annotate).toHaveBeenCalledWith('Test: Nested outer outer passing test') + const testPre = trackEvent.mock.calls.findIndex(([state]) => state === TestFrameworkState.TEST) + expect(annotate.mock.invocationCallOrder[0]).toBeLessThan(trackEvent.mock.invocationCallOrder[testPre]) + }) + + it('does not annotate mocha tests on the CLI flow', async () => { + const { annotate } = await runBeforeTest('mocha', { title: 't', parent: 'suite' }) + expect(annotate).not.toHaveBeenCalled() + }) +}) + +describe('service.before on the CLI flow (S6)', () => { + const registeredEvents = async (framework: string) => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + cliWith(null) + const browser = { on: vi.fn(), sessionId: 's1', capabilities: {}, config: {}, execute: vi.fn(), executeScript: vi.fn() } as any + const service = new BrowserstackService({} as any, [{}] as any, { user: 'foo', key: 'bar', framework, capabilities: {} } as any) + await service.before(service['_config'] as any, [], browser) + delete process.env.BROWSERSTACK_OBSERVABILITY + return vi.mocked(browser.on).mock.calls.map(([event]: [string]) => event) + } + + it('registers command and result for jasmine', async () => { + const events = await registeredEvents('jasmine') + expect(events).toContain('command') + expect(events).toContain('result') + }) + + it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { + const events = await registeredEvents(framework) + expect(events).toContain('result') + expect(events).not.toContain('command') + }) +}) + +describe('insights-handler.browserCommand on the CLI flow (S6)', () => { + const browser = { on: vi.fn(), sessionId: 's', capabilities: {}, config: {}, execute: vi.fn() } as any + const command = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/title', body: {} } + const result = { ...command, result: { value: 'StackDemo' } } + + const handlerFor = (framework: string) => { + const handler = new InsightsHandler(browser, framework) + handler['getIdentifier'] = vi.fn().mockReturnValue('Nested outer outer passing test') + handler['_tests'] = { 'Nested outer outer passing test': { uuid: 'spec-uuid' } } + return handler + } + + beforeEach(() => { + delete process.env[TESTOPS_SCREENSHOT_ENV] + }) + + it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).not.toHaveBeenCalled() + expect(trackEvent).toHaveBeenCalledTimes(1) + const [state, hook, { logEntry }] = trackEvent.mock.calls[0] + expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) + expect(logEntry.kind).toBe('HTTP') + expect(logEntry.test_run_uuid).toBe('spec-uuid') + expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + }) + + it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).toHaveBeenCalledWith([{ + test_run_uuid: 'spec-uuid', + timestamp: expect.any(String), + kind: 'HTTP', + http_response: { path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } } + }]) + }) + + it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { + process.env[TESTOPS_SCREENSHOT_ENV] = 'true' + const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } + + const jasmineTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: jasmineTrack }) + await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) + getInstanceSpy!.mockRestore() + + const mochaTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: mochaTrack }) + await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) + + expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) + expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) + }) + + it('drops commands with no spec yet, as legacy did before the first spec', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + handler['getIdentifier'] = vi.fn().mockReturnValue(undefined) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, {} as any) + await handler.browserCommand('client:afterCommand', { ...result } as any, {} as any) + + expect(trackEvent).not.toHaveBeenCalled() + }) +}) From 6b687de5fda0130235a1087e39faf9d20c2a96a9 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 17:05:33 +0530 Subject: [PATCH 05/15] feat(browserstack-service): jasmine Automate session verdict on CLI (SDK-7772) - WdioJasmineTestFramework keeps the legacy service.after() inputs: specs ran, the runner's failed-spec count, and the test and hook failure reasons of the live session in the order they happened - AutomateModule marks a jasmine worker's live session with that verdict, in the legacy body shape {status, name, reason}; a session no spec registered is still marked and a reloaded session keeps the mark onReload sent - ignoreHooksStatus is read from the worker's service options, because the binary config echo carries testObservabilityOptions empty - other frameworks keep the existing per-test aggregation Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 104 ++++++++++++++++++ .../src/cli/modules/automateModule.ts | 52 ++++++++- .../tests/cli/modules/automateModule.test.ts | 102 +++++++++++++++++ .../cli/wdioJasmineTestFramework.test.ts | 83 ++++++++++++++ 4 files changed, 338 insertions(+), 3 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 649e8366..7b946760 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -5,6 +5,8 @@ import type { Frameworks } from '@wdio/types' import type { HookStats, TestStats } from '@wdio/reporter' import TestFramework from './testFramework.js' +import AutomationFramework from './automationFramework.js' +import { AutomationFrameworkConstants } from './constants/automationFrameworkConstants.js' import { TestFrameworkState } from '../states/testFrameworkState.js' import { HookState } from '../states/hookState.js' import TestFrameworkInstance from '../instances/testFrameworkInstance.js' @@ -23,6 +25,17 @@ export interface JasmineSuiteContext { suiteFile?: string } +/** The Automate verdict legacy `service.after()` marks a session with. */ +export interface SessionVerdict { + status: 'passed' | 'failed' + reason?: string +} + +interface SessionFailure { + reason: string + fromHook: boolean +} + interface FailureFields { failure?: Array<{ backtrace: string[] }> failureReason?: string @@ -58,6 +71,12 @@ export default class WdioJasmineTestFramework extends TestFramework { static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' static #pendingEvents = 0 + static #serviceHookStates = new Set([ + TestFrameworkState.BEFORE_ALL, + TestFrameworkState.BEFORE_EACH, + TestFrameworkState.AFTER_EACH, + TestFrameworkState.AFTER_ALL, + ]) static #hookTypes = new Map([ ['beforeAll', 'BEFORE_ALL'], ['afterAll', 'AFTER_ALL'], @@ -78,9 +97,17 @@ export default class WdioJasmineTestFramework extends TestFramework { #openHook: TestFrameworkInstance | null = null #lastSpec: TestFrameworkInstance | null = null #queue: Promise = Promise.resolve() + // Legacy `service.ts` session verdict inputs: `_specsRan` and the runner result are per worker, + // the reason lists per session (`onReload` resets them). + #specsRan = false + #runnerFailedCount = 0 + // In the order they happened, as legacy pushed them + #sessionFailures = new Map() + static #current: WdioJasmineTestFramework | null = null constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { super(testFrameworks, testFrameworkVersions, binSessionId) + WdioJasmineTestFramework.#current = this logger.debug('WdioJasmineTestFramework: constructed') } @@ -109,6 +136,12 @@ export default class WdioJasmineTestFramework extends TestFramework { } try { await this.#queue + if (WdioJasmineTestFramework.#serviceHookStates.has(testFrameworkState)) { + if (hookState === HookState.POST) { + this.#recordHookResult(args.result as Frameworks.TestResult | undefined) + } + return + } if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { return } @@ -124,6 +157,7 @@ export default class WdioJasmineTestFramework extends TestFramework { if (hookState === HookState.PRE) { this.#suiteTitles.set(instance, args.suiteTitle) } else { + this.#recordTestResult(args.result as Frameworks.TestResult | undefined) // afterTest hands over the raw `Jasmine__TopLevel__Suite`; the modules need the describe chain beforeTest derived moduleArgs.suiteTitle = this.#suiteTitles.get(instance) ?? args.suiteTitle this.#suiteTitles.delete(instance) @@ -187,6 +221,9 @@ export default class WdioJasmineTestFramework extends TestFramework { const state = testStats.state const error = testStats.error + if (state === 'failed') { + this.#runnerFailedCount++ + } let result: string = state let failure: FailureFields = {} if (state === 'failed') { @@ -358,6 +395,73 @@ export default class WdioJasmineTestFramework extends TestFramework { } } + /** + * The verdict legacy `service.after()` would mark the live session with: passed only when the runner + * reported no failed spec, a spec ran, and no test or hook failed, with `ignoreHooksStatus` handled as + * coded there. `null` for any other session: legacy marked a reloaded session once, in `onReload`. + * `undefined` when this worker does not run jasmine on the CLI flow. + */ + static sessionVerdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null | undefined { + const current = WdioJasmineTestFramework.#current + return current ? current.#verdict(sessionId, ignoreHooksStatus) : undefined + } + + #verdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null { + if (!sessionId || sessionId !== WdioJasmineTestFramework.#liveSessionId()) { + return null + } + const failures = this.#sessionFailures.get(sessionId) ?? [] + const pureTestFailReasons = failures.filter(f => !f.fromHook).map(f => f.reason) + const hookFailReasons = failures.filter(f => f.fromHook).map(f => f.reason) + const failReasons = ignoreHooksStatus ? pureTestFailReasons : failures.map(f => f.reason) + const joined = (reasons: string[]) => reasons.length > 0 ? reasons.join('\n') : undefined + + if (this.#runnerFailedCount === 0 && this.#specsRan) { + const reasons = ignoreHooksStatus ? pureTestFailReasons : failReasons + return reasons.length > 0 ? { status: 'failed', reason: joined(reasons) } : { status: 'passed' } + } + if (ignoreHooksStatus && this.#specsRan) { + const hasOnlyHookFailures = failReasons.length === 0 && hookFailReasons.length > 0 + if (hasOnlyHookFailures && pureTestFailReasons.length === 0) { + return { status: 'passed' } + } + return { status: 'failed', reason: joined(pureTestFailReasons) } + } + return { + status: 'failed', + reason: ignoreHooksStatus && pureTestFailReasons.length > 0 ? joined(pureTestFailReasons) : joined(failReasons), + } + } + + /** Legacy `service.afterTest()`: the spec ran; a failure counts toward the session. */ + #recordTestResult(result: Frameworks.TestResult | undefined) { + this.#specsRan = true + if (!result || result.passed || result.skipped) { + return + } + this.#recordFailure((result.error && result.error.message) || 'Unknown Error', false) + } + + /** Legacy `service.afterHook()`, any of the four hook types. */ + #recordHookResult(result: Frameworks.TestResult | undefined) { + if (!result || result.passed) { + return + } + this.#recordFailure((result.error && result.error.message) || 'Hook failed', true) + } + + #recordFailure(reason: string, fromHook: boolean) { + const sessionId = WdioJasmineTestFramework.#liveSessionId() + const failures = this.#sessionFailures.get(sessionId) ?? [] + failures.push({ reason, fromHook }) + this.#sessionFailures.set(sessionId, failures) + } + + static #liveSessionId(): string { + const autoInstance = AutomationFramework.getTrackedInstance() + return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' + } + #createInstance() { const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) const instance = new TestFrameworkInstance( diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index d01d3080..dbde960e 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -1,4 +1,6 @@ import BaseModule from './baseModule.js' +import WdioJasmineTestFramework from '../frameworks/wdioJasmineTestFramework.js' +import { BrowserstackCLI } from '../index.js' import { BStackLogger } from '../cliLogger.js' import TestFramework from '../frameworks/testFramework.js' import { TestFrameworkState } from '../states/testFrameworkState.js' @@ -348,6 +350,16 @@ export default class AutomateModule extends BaseModule { const accessKey = this.config.accessKey as string const testContextOptions = this.config.testContextOptions as TestContextOptions + // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options + const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } + const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true + const liveSessionId = this.liveSessionId() + if (WdioJasmineTestFramework.sessionVerdict(liveSessionId, ignoreHooksStatus) !== undefined) { + await this.markJasmineSessions(liveSessionId, ignoreHooksStatus) + this.sessionMap.clear() + return + } + for (const [sessionId, sessionData] of this.sessionMap.entries()) { try { const failedTests = Array.from(sessionData.testResults.values()).filter(test => test.status === 'failed') @@ -389,6 +401,39 @@ export default class AutomateModule extends BaseModule { this.sessionMap.clear() } + private liveSessionId(): string { + const autoInstance = AutomationFramework.getTrackedInstance() + return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' + } + + /** + * Jasmine: legacy `service.after()` marked only the live session, with the worker's status, the last + * name, and the test and hook failure reasons. A session nothing registered (a beforeAll failed before + * any spec ran) is still marked, and a reloaded one is left to the mark `onReload` already sent. + */ + private async markJasmineSessions(liveSessionId: string, ignoreHooksStatus: boolean) { + const testContextOptions = this.config.testContextOptions as TestContextOptions + const auth = { user: this.config.userName as string, key: this.config.accessKey as string } + const sessionIds = new Set(this.sessionMap.keys()) + if (liveSessionId && isBrowserstackSession(AutomationFramework.getDriver(AutomationFramework.getTrackedInstance()) as WebdriverIO.Browser)) { + sessionIds.add(liveSessionId) + } + + for (const sessionId of sessionIds) { + try { + await this.flushSessionName(sessionId) + const verdict = WdioJasmineTestFramework.sessionVerdict(sessionId, ignoreHooksStatus) + if (!verdict || testContextOptions.skipSessionStatus) { + continue + } + const name = testContextOptions.skipSessionName ? undefined : this.sessionMap.get(sessionId)?.lastTestName || undefined + await this.markSessionStatus(sessionId, verdict.status, verdict.reason, auth, name) + } catch (error) { + this.logger.error(`Failed to process session ${sessionId}: ${error}`) + } + } + } + // An App Automate session is identified by the service-level app / skipAppOverride flag, // OR an app supplied only via the appium:app / appium:options.app capability — which the // service-level config (this.config) does not carry. Mirrors accessibilityModule.isAppAutomateSession. @@ -474,10 +519,10 @@ export default class AutomateModule extends BaseModule { )(sessionId, sessionName, config) } - async markSessionStatus(sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }): Promise { + async markSessionStatus(sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }, sessionName?: string): Promise { return await PerformanceTester.measureWrapper( PERFORMANCE_SDK_EVENTS.AUTOMATE_EVENTS.SESSION_STATUS, - async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }) => { + async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }, sessionName?: string) => { try { const auth = Buffer.from(`${config.user}:${config.key}`).toString('base64') const { url: sessionStatusApiUrl, method, product } = this.resolveSessionApi(sessionId) @@ -485,6 +530,7 @@ export default class AutomateModule extends BaseModule { const body = { status: sessionStatus, + ...(sessionName ? { name: sessionName } : {}), ...(sessionErrorMessage ? { reason: sessionErrorMessage } : {}) } @@ -504,7 +550,7 @@ export default class AutomateModule extends BaseModule { this.logger.error(`Failed to update session status on BrowserStack: ${err}`) } } - )(sessionId, sessionStatus, sessionErrorMessage, config) + )(sessionId, sessionStatus, sessionErrorMessage, config, sessionName) } } diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index b0e27c76..098b946a 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -9,6 +9,8 @@ import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/te import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' +import WdioJasmineTestFramework from '../../../src/cli/frameworks/wdioJasmineTestFramework.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' import type { Options } from '@wdio/types' // Mock dependencies @@ -21,6 +23,19 @@ vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ } })) +vi.mock('../../../src/cli/index.js', () => ({ + BrowserstackCLI: { + getInstance: vi.fn(() => ({ options: {} })) + } +})) + +// undefined = this worker is not jasmine on the CLI flow, so the existing aggregation applies +vi.mock('../../../src/cli/frameworks/wdioJasmineTestFramework.js', () => ({ + default: { + sessionVerdict: vi.fn(() => undefined) + } +})) + vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ default: { getTrackedInstance: vi.fn(), @@ -1005,3 +1020,90 @@ describe('AutomateModule preferScenarioName', () => { expect(namesPUT()).toContain('Can log in') }) }) + +describe('AutomateModule — jasmine session verdict', () => { + let automateModule: AutomateModule + const putBodies = () => vi.mocked(fetch).mock.calls.map(([url, opts]) => [String(url).split('/sessions/')[1], JSON.parse((opts as { body: string }).body)]) + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 'live' } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 'live' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'u', + accessKey: 'k' + } as any + }) + + afterEach(() => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReset().mockReturnValue(undefined) + }) + + it('marks the live session with the framework verdict, the last name and the joined reasons', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' + ? { status: 'failed', reason: 'beforeAll hook failure\nafterEach hook failure' } + : null) + ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Hookfail afterEach suite', appliedName: 'Hookfail afterEach suite', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([ + ['live.json', { status: 'failed', name: 'Hookfail afterEach suite', reason: 'beforeAll hook failure\nafterEach hook failure' }] + ]) + }) + + it('marks a live session no spec registered, without a name', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'beforeAll hook failure' }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'beforeAll hook failure' }]]) + }) + + it('leaves a reloaded session to its onReload mark, but still names it', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' ? { status: 'passed' } : null) + ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([ + ['old.json', { name: 'Suite A' }], + ['live.json', { status: 'passed' }], + ]) + }) + + it('passes ignoreHooksStatus from the worker\'s service options to the verdict', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'passed' }) + vi.mocked(BrowserstackCLI.getInstance).mockReturnValueOnce({ options: { testObservabilityOptions: { ignoreHooksStatus: true } } } as any) + + await automateModule.onAfterExecute() + + expect(WdioJasmineTestFramework.sessionVerdict).toHaveBeenCalledWith('live', true) + }) + + it('honours skipSessionStatus and skipSessionName', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'x' }) + ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', testResults: new Map(), scenariosRan: 0 }) + ;(automateModule.config as any).testContextOptions = { skipSessionName: true, skipSessionStatus: true } + + await automateModule.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('keeps the existing per-test aggregation when the worker is not jasmine', async () => { + (automateModule as any).sessionMap.set('live', { + lastTestName: 'Suite', appliedName: 'Suite', scenariosRan: 0, + testResults: new Map([['t', { testName: 'Suite', status: 'failed', reason: 'boom' }]]) + }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'boom' }]]) + }) +}) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index cfce5f23..dd1e3b9b 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -8,6 +8,7 @@ import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' import { HookState } from '../../src/cli/states/hookState.js' import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' import TestHubModule from '../../src/cli/modules/testHubModule.js' +import AutomationFramework from '../../src/cli/frameworks/automationFramework.js' vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) @@ -396,6 +397,88 @@ describe('WdioJasmineTestFramework', () => { expect(WdioJasmineTestFramework.isIdle()).toBe(true) }) + describe('session verdict (legacy service.after())', () => { + let liveSession = 'live' + const spec = (title: string, overrides: Record = {}) => testStats({ uid: title, title, fullTitle: `Suite ${title}`, ...overrides }) + const reporterEnd = (stats: ReturnType, state: string) => + framework.onReporterTestEnd({ ...stats, state, end: new Date() } as any, context()) + const serviceTest = (title: string, result: Record) => + framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: { fullName: `Suite ${title}`, description: title }, result }) + const serviceHook = (state: State, result: Record) => framework.trackEvent(state, HookState.POST, { test: {}, result }) + const fail = (message: string) => ({ passed: false, error: new Error(message) }) + + beforeEach(() => { + liveSession = 'live' + vi.spyOn(AutomationFramework, 'getTrackedInstance').mockReturnValue({} as any) + vi.spyOn(AutomationFramework, 'getState').mockImplementation(() => liveSession) + }) + + it('reproduces the CP0 hookfail verdict: failed, every hook failure in the order it happened', async () => { + await serviceHook(TestFrameworkState.BEFORE_ALL, fail('beforeAll hook failure')) + for (const title of ['beforeAll child one', 'beforeAll child two', 'beforeEach child one', 'beforeEach child two']) { + if (title.startsWith('beforeEach')) { + await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) + } + const stats = spec(title) + framework.onReporterTestStart(stats as any, context()) + reporterEnd(stats, 'failed') + } + for (const title of ['afterEach child one', 'afterEach child two']) { + const stats = spec(title) + framework.onReporterTestStart(stats as any, context()) + await serviceTest(title, { passed: true }) + await serviceHook(TestFrameworkState.AFTER_EACH, fail('afterEach hook failure')) + reporterEnd(stats, 'failed') + } + await drain() + + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ + status: 'failed', + reason: 'beforeAll hook failure\nbeforeEach hook failure\nbeforeEach hook failure\nafterEach hook failure\nafterEach hook failure' + }) + }) + + it('fails a session where no spec ran, even with no failure recorded', async () => { + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('passes when specs ran and nothing failed, with no reason', async () => { + const stats = spec('ok') + framework.onReporterTestStart(stats as any, context()) + await serviceTest('ok', { passed: true }) + reporterEnd(stats, 'passed') + await drain() + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) + }) + + it('does not count a pending() spec as a failure', async () => { + await serviceTest('pending', { passed: false, skipped: true }) + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) + }) + + it('under ignoreHooksStatus passes a run whose only failures are hooks, and keeps test failures', async () => { + await serviceTest('ok', { passed: true }) + await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) + const child = spec('child') + framework.onReporterTestStart(child as any, context()) + reporterEnd(child, 'failed') + await drain() + expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'passed' }) + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: 'beforeEach hook failure' }) + + await serviceTest('bad', fail('boom')) + expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('keeps failures per session and gives no verdict for a session that is no longer live', async () => { + await serviceTest('first', fail('before reload')) + liveSession = 'reloaded' + await serviceTest('second', { passed: true }) + expect(WdioJasmineTestFramework.sessionVerdict('reloaded', false)).toEqual({ status: 'passed' }) + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toBeNull() + }) + }) + // The class calls these TestHubModule methods directly; renaming or removing one must fail here. it('relies on TestHubModule handlers that exist', () => { for (const method of ['onBeforeTest', 'sendTestFrameworkEvent', 'sendLogCreatedEvent']) { From 4c3c6ea56cf222f9d2f299d1b2da3e4a5d649c1a Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 22:21:04 +0530 Subject: [PATCH 06/15] fix(browserstack-service): strip SDK-only session-name options from capabilities (port of 3cdbfec, SDK-7772) Port of 3cdbfec from SDK-7606/wdio-cucumber-platformisation-v8. sessionNamePrependTopLevelSuiteTitle and sessionNameOmitTestTitle reach the outgoing capabilities on the CLI flow and the hub rejects the session ("additional properties [...] outside of the schema"). Both are read from testContextOptions, so stripping them from the capabilities does not disable either option. Co-Authored-By: Claude Opus 5.5 --- packages/browserstack-service/src/constants.ts | 2 +- packages/browserstack-service/tests/skipAppOverride.test.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/browserstack-service/src/constants.ts b/packages/browserstack-service/src/constants.ts index ae000252..1132ba04 100644 --- a/packages/browserstack-service/src/constants.ts +++ b/packages/browserstack-service/src/constants.ts @@ -42,7 +42,7 @@ export const DEFAULT_WAIT_TIMEOUT_FOR_PENDING_UPLOADS = 5000 // 5s export const DEFAULT_WAIT_INTERVAL_FOR_PENDING_UPLOADS = 100 // 100ms export const BSTACK_SERVICE_VERSION = bstackServiceVersion -export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName'] +export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName', 'sessionNamePrependTopLevelSuiteTitle', 'sessionNameOmitTestTitle'] export const BROWSERSTACK_TEST_PLAN_ID = 'BROWSERSTACK_TEST_PLAN_ID' export const LOGS_FILE = 'logs/bstack-wdio-service.log' diff --git a/packages/browserstack-service/tests/skipAppOverride.test.ts b/packages/browserstack-service/tests/skipAppOverride.test.ts index 5b175848..f65e36e8 100644 --- a/packages/browserstack-service/tests/skipAppOverride.test.ts +++ b/packages/browserstack-service/tests/skipAppOverride.test.ts @@ -89,7 +89,9 @@ describe('NOT_ALLOWED_KEYS_IN_CAPS cloud-leak strip', () => { expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('skipAppOverride') }) - it('includes preferScenarioName so it is never forwarded to bstack:options', () => { + it('includes the SDK-only options the binary round-trips into caps on the CLI flow', () => { expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('preferScenarioName') + expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('sessionNamePrependTopLevelSuiteTitle') + expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('sessionNameOmitTestTitle') }) }) From 4343a4c55c7993186ee8c214751ee6ba4f3ccb44 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 22:21:46 +0530 Subject: [PATCH 07/15] fix(browserstack-service): apply jasmine sessionNameFormat on CLI (SDK-7772) The binary config echo cannot carry the sessionNameFormat function, so it arrives empty and the session keeps its default name. For jasmine the AutomateModule now takes the function from the worker's service options, called as legacy did (no test title). Other frameworks are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/automateModule.ts | 26 ++++++++++-- .../tests/cli/modules/automateModule.test.ts | 42 +++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index dbde960e..c6554f50 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -90,9 +90,10 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + const sessionNameFormat = this.sessionNameFormatFor(instace, testContextOptions) + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -183,9 +184,10 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + const sessionNameFormat = this.sessionNameFormatFor(instace, testContextOptions) + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -326,6 +328,22 @@ export default class AutomateModule extends BaseModule { } } + /** + * The binary's config echo cannot carry a function, so `sessionNameFormat` arrives empty. Jasmine + * takes it from the worker's own service options, as legacy did. + */ + private sessionNameFormatFor(instance: TestFrameworkInstance, testContextOptions: TestContextOptions) { + if (testContextOptions.sessionNameFormat) { + return testContextOptions.sessionNameFormat + } + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + if (!frameworkName.toLowerCase().includes('jasmine')) { + return undefined + } + const format = (BrowserstackCLI.getInstance().options as { sessionNameFormat?: unknown })?.sessionNameFormat + return typeof format === 'function' ? format as TestContextOptions['sessionNameFormat'] : undefined + } + private isCucumberInstance(instance: TestFrameworkInstance): boolean { const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') return frameworkName.toLowerCase().includes('cucumber') diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 098b946a..4d1fbf45 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -1107,3 +1107,45 @@ describe('AutomateModule — jasmine session verdict', () => { expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'boom' }]]) }) }) + +describe('AutomateModule — jasmine sessionNameFormat', () => { + let automateModule: AutomateModule + const format = vi.fn((_config: unknown, _caps: unknown, suiteTitle: string, testTitle?: string) => `fmt[${suiteTitle}][${String(testTitle)}]`) + const names = () => vi.mocked(fetch).mock.calls.map(([, opts]) => JSON.parse((opts as { body: string }).body).name) + + const runBeforeTest = (frameworkName: string, test: Record) => { + vi.mocked(TestFramework.getState).mockImplementation((_i, key) => key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME ? frameworkName : undefined) + return automateModule.onBeforeTest({ instance: {}, test, suiteTitle: 'Nested outer' }) + } + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 's1' } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 's1' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + vi.mocked(BrowserstackCLI.getInstance).mockReturnValue({ options: { sessionNameFormat: format } } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false, sessionNameFormat: '' }, + userName: 'u', + accessKey: 'k' + } as any + }) + + afterEach(() => { + vi.mocked(BrowserstackCLI.getInstance).mockReset().mockReturnValue({ options: {} } as any) + }) + + it('applies the service-option format for jasmine, with no test title (legacy call shape)', async () => { + await runBeforeTest('WebdriverIO-jasmine', { description: 'outer passing test', fullName: 'Nested outer outer passing test' }) + expect(names()).toEqual(['fmt[Nested outer][undefined]']) + }) + + it('leaves mocha on the binary config (format absent there)', async () => { + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(format).not.toHaveBeenCalled() + expect(names()).toEqual(['Suite - t']) + }) +}) From f97f8d9f74dd3bb210270f4b342b03843e2a85c4 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 23:14:02 +0530 Subject: [PATCH 08/15] feat(browserstack-service): jasmine accessibility gate on CLI (SDK-7772) - no scan window before the first jasmine spec, as the classic handler (PRE_TEST_SCAN_FRAMEWORKS excludes jasmine); mocha/cucumber keep it - the include/exclude filter and the test name read the jasmine spec description when the test has no title Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/accessibilityModule.ts | 16 +++++-- .../cli/modules/accessibilityModule.test.ts | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index edcb1b3b..d8cf2a26 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -278,7 +278,7 @@ export default class AccessibilityModule extends BaseModule { // on suite and test titles, and in this window neither exists yet. onBeforeTest // re-computes the per-test gate, tags included, so this only affects the window. const preTestSessionId = this.currentSessionId() - if (this.autoScanning && preTestSessionId !== undefined && preTestSessionId !== null) { + if (this.autoScanning && this.supportsPreTestWindow() && preTestSessionId !== undefined && preTestSessionId !== null) { this.accessibilityMap.set(preTestSessionId, true) this.logger.debug('Accessibility scan gate opened ahead of the first test') } @@ -338,9 +338,11 @@ export default class AccessibilityModule extends BaseModule { try { this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.') const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' - const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string, description?: string } : {}) || {} + // jasmine specs carry their name in `description` (`title` is unset), as on the classic handler + const testTitle = test.title ?? test.description - this.currentTestName = test.title || null + this.currentTestName = testTitle || null this.testContextSeen = true const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() @@ -353,7 +355,7 @@ export default class AccessibilityModule extends BaseModule { // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg // behaviour — both extra args arrive undefined/false and the tag branch is not taken. const world = args.world as { [key: string]: unknown } | undefined - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility + const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, testTitle || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility this.accessibilityMap.set(sessionId, shouldScanTest) @@ -537,6 +539,12 @@ export default class AccessibilityModule extends BaseModule { return false } + // Legacy jasmine opened no scan window before its first spec (classic PRE_TEST_SCAN_FRAMEWORKS) + private supportsPreTestWindow(): boolean { + const frameworks = BrowserstackCLI.getInstance().getTestFramework()?.getTestFrameworks() ?? [] + return !frameworks.some((name) => String(name).toLowerCase().includes('jasmine')) + } + // See the classic handler: one definition, used at every scan site. private get hasNoParent(): boolean { return !this.currentHookRunUuid && !this.testContextSeen diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index d4114756..5349c5e9 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -59,6 +59,7 @@ import AutomationFramework from '../../../src/cli/frameworks/automationFramework import { AutomationFrameworkState } from '../../../src/cli/states/automationFrameworkState.js' import { HookState } from '../../../src/cli/states/hookState.js' import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' describe('AccessibilityModule', () => { let accessibilityModule: AccessibilityModule @@ -194,6 +195,30 @@ describe('AccessibilityModule', () => { expect(accessibilityModule.accessibilityMap.get('session-w')).toBe(true) }) + it('opens no window for jasmine, whose classic flow scanned nothing before the first spec', async () => { + withA11yCaps() + const cliSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + getTestFramework: () => ({ getTestFrameworks: () => ['WebdriverIO-jasmine'] }) + } as any) + + await accessibilityModule.onBeforeExecute() + + expect(accessibilityModule.accessibilityMap.has('session-w')).toBe(false) + cliSpy.mockRestore() + }) + + it('still opens the window for mocha', async () => { + withA11yCaps() + const cliSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + getTestFramework: () => ({ getTestFrameworks: () => ['WebdriverIO-mocha'] }) + } as any) + + await accessibilityModule.onBeforeExecute() + + expect(accessibilityModule.accessibilityMap.get('session-w')).toBe(true) + cliSpy.mockRestore() + }) + it('respects autoScanning — the one validation the window still owns', async () => { withA11yCaps() accessibilityModule.autoScanning = false @@ -276,6 +301,25 @@ describe('AccessibilityModule', () => { }) }) + describe('onBeforeTest — jasmine spec name', () => { + it('filters on the spec description when the test has no title, and names the test with it', async () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockImplementation(() => 'session-w') + await accessibilityModule.onBeforeTest({ + suiteTitle: 'Nested outer', + test: { description: 'outer passing test', fullName: 'Nested outer outer passing test' } + }) + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Nested outer', 'outer passing test', {}, undefined, false) + }) + + it('keeps the title when there is one (mocha)', async () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockImplementation(() => 'session-w') + await accessibilityModule.onBeforeTest({ suiteTitle: 'Suite', test: { title: 't', description: 'ignored' } }) + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Suite', 't', {}, undefined, false) + }) + }) + describe('onBeforeExecute', () => { it('should patch browser methods when automation instance exists', async () => { vi.mocked(AutomationFramework.getState).mockImplementation((instance, key) => { From ddc719db6158846ed35790ccd86aaf705694b9d1 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sun, 27 Sep 2026 00:24:34 +0530 Subject: [PATCH 09/15] fix(browserstack-service): name the current jasmine test for Percy on CLI (SDK-7772) PercySDK attaches testCase from TestReporter.currentTest.name, which legacy sets while building the TestRunStarted event. The jasmine CLI branch of onTestStart returns before that, so CLI snapshots carried no testCase. It now sets the name next to the CLI uuid. Mocha is unchanged. Co-Authored-By: Claude Opus 5.5 --- packages/browserstack-service/src/reporter.ts | 1 + .../browserstack-service/tests/reporter.jasmineCli.test.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index b593e5c8..c4d839e3 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -211,6 +211,7 @@ class _TestReporter extends WDIOReporter { const cliUuid = this.cliJasmineFramework()?.onReporterTestStart(testStats, this.jasmineSuiteContext()) if (cliUuid) { _TestReporter.currentTest.uuid = cliUuid + _TestReporter.currentTest.name = testStats.title _TestReporter._tests[testStats.fullTitle] = { uuid: cliUuid } } return diff --git a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts index 537c8b55..76ea1a68 100644 --- a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts +++ b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts @@ -111,6 +111,11 @@ describe('reporter jasmine arm', () => { expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) }) + it('names the current test for Percy\'s testCase, as legacy getRunData did', async () => { + await reporter.onTestStart(testStats() as any) + expect((TestReporter as any).currentTest).toMatchObject({ uuid: 'cli-uuid', name: 'outer passing test' }) + }) + it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { const stats = testStats() as Record await reporter.onTestEnd(stats as any) From d54cfc1e55b2834c18d8f48efd15f1363e8368bc Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Mon, 28 Sep 2026 12:19:16 +0530 Subject: [PATCH 10/15] fix(browserstack-service): forward the CLI-arg test plan id to the binary (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../browserstack-service/src/cli/cliUtils.ts | 8 +-- .../tests/cli/cliUtils.test.ts | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 6fbe10bf..6ad4a907 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -23,6 +23,7 @@ import { isFalse, isTurboScale, shouldAddServiceVersion, + getTestPlanId, } from '../util.js' import PerformanceTester from '../instrumentation/performance/performance-tester.js' import { EVENTS as PerformanceEvents } from '../instrumentation/performance/constants.js' @@ -32,7 +33,6 @@ import type { Options, Capabilities } from '@wdio/types' import type { BrowserstackConfig, BrowserstackOptions, - TestManagementOptions, TestObservabilityOptions, } from '../types.js' import { TestFrameworkConstants } from './frameworks/constants/testFrameworkConstants.js' @@ -116,11 +116,7 @@ export class CLIUtils { ) const observabilityOptions: TestObservabilityOptions = options.testObservabilityOptions || {} - const testManagementOptions: TestManagementOptions = - options.testManagementOptions || {} - const testPlanId = typeof testManagementOptions.testPlanId === 'string' - ? testManagementOptions.testPlanId.trim() - : '' + const testPlanId = (getTestPlanId(options as BrowserstackConfig & Options.Testrunner) || '').trim() const binconfig: Record = { userName: observabilityOptions.user || config.user, accessKey: observabilityOptions.key || config.key, diff --git a/packages/browserstack-service/tests/cli/cliUtils.test.ts b/packages/browserstack-service/tests/cli/cliUtils.test.ts index aa82167c..5898bae7 100644 --- a/packages/browserstack-service/tests/cli/cliUtils.test.ts +++ b/packages/browserstack-service/tests/cli/cliUtils.test.ts @@ -226,6 +226,56 @@ describe('CLIUtils', () => { testPlanId: 'tm-plan-456' }) }) + + describe('test plan id from env and CLI args', () => { + const CLI_ARG = '--browserstack.testManagementOptions.testPlanId' + const capabilities = [{ browserName: 'chrome' }] + let originalArgv: string[] + + beforeEach(() => { + originalArgv = process.argv + delete process.env.BROWSERSTACK_TEST_PLAN_ID + }) + + afterEach(() => { + process.argv = originalArgv + delete process.env.BROWSERSTACK_TEST_PLAN_ID + }) + + const planOf = (options = createBrowserstackOptions()) => + JSON.parse(CLIUtils.getBinConfig(mockConfig, capabilities, options)).testManagementOptions + + it('reads BROWSERSTACK_TEST_PLAN_ID', () => { + process.env.BROWSERSTACK_TEST_PLAN_ID = 'env-plan-1' + expect(planOf()).toEqual({ testPlanId: 'env-plan-1' }) + }) + + it('reads the CLI arg followed by its value', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', CLI_ARG, 'TP-123'] + expect(planOf()).toEqual({ testPlanId: 'TP-123' }) + }) + + it('reads the CLI arg in --key=value form', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', `${CLI_ARG}=12345`] + expect(planOf()).toEqual({ testPlanId: '12345' }) + }) + + it('prefers the CLI arg over the service option', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', CLI_ARG, 'cli-plan'] + expect(planOf(createBrowserstackOptions({ testManagementOptions: { testPlanId: 'option-plan' } }))) + .toEqual({ testPlanId: 'cli-plan' }) + }) + + it('omits testManagementOptions when no source sets an id', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js'] + expect(planOf()).toBeUndefined() + }) + + it('omits testManagementOptions when the env id is blank', () => { + process.env.BROWSERSTACK_TEST_PLAN_ID = ' ' + expect(planOf()).toBeUndefined() + }) + }) }) describe('getSdkVersion', () => { From bcea0e9701cf3fcf68e63a813e99efaee4b699bc Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Mon, 28 Sep 2026 13:29:01 +0530 Subject: [PATCH 11/15] fix(browserstack-service): stop forwarding the raw test plan id arg to the binary (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../src/cli/grpcClient.ts | 23 +++++++++++++- .../tests/cli/grpcClient.test.ts | 30 ++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/packages/browserstack-service/src/cli/grpcClient.ts b/packages/browserstack-service/src/cli/grpcClient.ts index 2aabc1cd..909dd160 100644 --- a/packages/browserstack-service/src/cli/grpcClient.ts +++ b/packages/browserstack-service/src/cli/grpcClient.ts @@ -55,6 +55,27 @@ const SUMMARY_ANSI = { warn: { base: '\x1b[33m', emphasis: '\x1b[1;33m' } } +const TEST_PLAN_ID_ARG = '--browserstack.testManagementOptions.testPlanId' + +// The binary parses CLI args as `--key=value` only, so the space form reaches it as `true` and +// overrides the id getBinConfig already resolved; the id travels in the bin config instead. +export function stripTestPlanIdArg(args: string[]): string[] { + const result: string[] = [] + for (let i = 0; i < args.length; i++) { + if (args[i] === TEST_PLAN_ID_ARG) { + if (i + 1 < args.length && !args[i + 1].startsWith('-')) { + i++ + } + continue + } + if (args[i].startsWith(`${TEST_PLAN_ID_ARG}=`)) { + continue + } + result.push(args[i]) + } + return result +} + /** * GrpcClient - Singleton class for managing gRPC client connections * @@ -180,7 +201,7 @@ export class GrpcClient { sdkVersion: packageVersion, pathProject: process.cwd(), pathConfig: path.resolve(process.cwd(), 'browserstack.yml'), - cliArgs: process.argv.slice(2), + cliArgs: stripTestPlanIdArg(process.argv.slice(2)), frameworks: [automationFrameworkDetail.name, testFrameworkDetail.name], frameworkVersions, language: CLIUtils.getSdkLanguage(), diff --git a/packages/browserstack-service/tests/cli/grpcClient.test.ts b/packages/browserstack-service/tests/cli/grpcClient.test.ts index 94312c1c..0bf8b808 100644 --- a/packages/browserstack-service/tests/cli/grpcClient.test.ts +++ b/packages/browserstack-service/tests/cli/grpcClient.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import { GrpcClient } from '../../src/cli/grpcClient.js' +import { GrpcClient, stripTestPlanIdArg } from '../../src/cli/grpcClient.js' import { BStackLogger } from '../../src/cli/cliLogger.js' vi.mock('../../src/grpc/index.js', () => ({ @@ -241,3 +241,31 @@ describe('GrpcClient.logCreatedEvent', () => { }) }) }) + +describe('stripTestPlanIdArg', () => { + const ARG = '--browserstack.testManagementOptions.testPlanId' + + it('drops the arg and its value in the space form', () => { + expect(stripTestPlanIdArg(['run', 'wdio.conf.js', ARG, 'TP-123', '--spec', 'a.js'])) + .toEqual(['run', 'wdio.conf.js', '--spec', 'a.js']) + }) + + it('drops the arg in the = form', () => { + expect(stripTestPlanIdArg(['run', `${ARG}=TP-123`, 'wdio.conf.js'])) + .toEqual(['run', 'wdio.conf.js']) + }) + + it('leaves args untouched and in order when the arg is absent', () => { + const args = ['run', 'wdio.conf.js', '--browserstack.buildName', 'b1', '--logLevel=info'] + expect(stripTestPlanIdArg(args)).toEqual(args) + }) + + it('keeps a following flag when the arg has no value', () => { + expect(stripTestPlanIdArg(['run', ARG, '--spec', 'a.js'])) + .toEqual(['run', '--spec', 'a.js']) + }) + + it('handles the arg as the last token', () => { + expect(stripTestPlanIdArg(['run', 'wdio.conf.js', ARG])).toEqual(['run', 'wdio.conf.js']) + }) +}) From d8f56c135d734edd20eb05784eb074d1b2dc28bb Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Tue, 29 Sep 2026 20:21:52 +0530 Subject: [PATCH 12/15] test(cli): fold skipTestHub marker cases into the TestHubModule suite (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../modules/testHubModule.skipMarker.test.ts | 57 ------------------- .../tests/cli/modules/testHubModule.test.ts | 35 ++++++++++++ 2 files changed, 35 insertions(+), 57 deletions(-) delete mode 100644 packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts deleted file mode 100644 index 3dbba978..00000000 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' - -import * as bstackLogger from '../../../src/bstackLogger.js' -import TestHubModule from '../../../src/cli/modules/testHubModule.js' -import TestFrameworkInstance from '../../../src/cli/instances/testFrameworkInstance.js' -import TrackedInstance from '../../../src/cli/instances/trackedInstance.js' -import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' -import { HookState } from '../../../src/cli/states/hookState.js' -import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -describe('TestHubModule skipTestHub marker', () => { - let module: TestHubModule - let sendTestFrameworkEvent: ReturnType - let sendTestSessionEvent: ReturnType - - const instanceIn = (state: State, hook: State, framework = 'WebdriverIO-cucumber') => { - const instance = new TestFrameworkInstance(TrackedInstance.createContext('t'), [framework], { [framework]: '9' }, state, hook) - instance.updateMultipleEntries({ - [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: framework, - [TestFrameworkConstants.KEY_TEST_UUID]: 'uuid-1', - [TestFrameworkConstants.KEY_TEST_RESULT_AT]: 'now', - }) - return instance - } - - beforeEach(() => { - module = new TestHubModule({}) - sendTestFrameworkEvent = vi.spyOn(module, 'sendTestFrameworkEvent').mockResolvedValue(true) - sendTestSessionEvent = vi.spyOn(module, 'sendTestSessionEvent').mockResolvedValue(undefined) - }) - - afterEach(() => { - vi.restoreAllMocks() - }) - - it('handles test and hook events that carry no marker, as before', () => { - module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.BEFORE_ALL, HookState.POST) }) - expect(sendTestSessionEvent).toHaveBeenCalledTimes(1) - expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(2) - }) - - it('ignores events marked skipTestHub', () => { - module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: true }) - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.POST), skipTestHub: true }) - expect(sendTestSessionEvent).not.toHaveBeenCalled() - expect(sendTestFrameworkEvent).not.toHaveBeenCalled() - }) - - it('only honours the literal true', () => { - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: 'true' }) - expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(1) - }) -}) diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts index 47316f20..c1031d39 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts @@ -152,6 +152,15 @@ describe('TestHubModule', () => { autoInstance: [mockAutomationInstance] }) }) + + it('should skip events marked skipTestHub', async () => { + const sendTestSessionEventSpy = vi.spyOn(testHubModule, 'sendTestSessionEvent').mockResolvedValue() + + await testHubModule.onBeforeTest({ test: { title: 'Test' } as Frameworks.Test, skipTestHub: true }) + + expect(AutomationFramework.getTrackedInstance).not.toHaveBeenCalled() + expect(sendTestSessionEventSpy).not.toHaveBeenCalled() + }) }) describe('onAllTestEvents', () => { @@ -225,6 +234,32 @@ describe('TestHubModule', () => { expect(sendTestFrameworkEventSpy).toHaveBeenCalledWith(mockArgs) }) + + it('should skip events marked skipTestHub', async () => { + const mockInstance = { + getCurrentTestState: vi.fn(() => TestFrameworkState.TEST), + getCurrentHookState: vi.fn(() => HookState.POST) + } + const sendTestFrameworkEventSpy = vi.spyOn(testHubModule, 'sendTestFrameworkEvent').mockResolvedValue() + + await testHubModule.onAllTestEvents({ instance: mockInstance, skipTestHub: true }) + + expect(mockInstance.getCurrentTestState).not.toHaveBeenCalled() + expect(sendTestFrameworkEventSpy).not.toHaveBeenCalled() + }) + + it('should only treat a literal true skipTestHub as the marker', async () => { + const mockInstance = { + getCurrentTestState: vi.fn(() => TestFrameworkState.TEST), + getCurrentHookState: vi.fn(() => HookState.PRE) + } + const sendTestFrameworkEventSpy = vi.spyOn(testHubModule, 'sendTestFrameworkEvent').mockResolvedValue() + const mockArgs = { instance: mockInstance, skipTestHub: 'true' } + + await testHubModule.onAllTestEvents(mockArgs) + + expect(sendTestFrameworkEventSpy).toHaveBeenCalledWith(mockArgs) + }) }) describe('sendTestFrameworkEvent', () => { From 977dbfef10c4c8c385115a024de6d89f4a86d7e9 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 30 Sep 2026 10:13:48 +0530 Subject: [PATCH 13/15] test: fold jasmine CLI cases into the existing cli, reporter, service and insights-handler suites (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../tests/cli/cliUtils.test.ts | 10 + .../tests/cli/index.test.ts | 38 +++ .../tests/cli/wdioJasmineGate.test.ts | 57 ---- .../tests/insights-handler.test.ts | 97 +++++++ .../tests/reporter.jasmineCli.test.ts | 193 ------------- .../tests/reporter.test.ts | 190 +++++++++++++ .../tests/service.jasmineCliGates.test.ts | 254 ------------------ .../tests/service.test.ts | 149 ++++++++++ 8 files changed, 484 insertions(+), 504 deletions(-) delete mode 100644 packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts delete mode 100644 packages/browserstack-service/tests/reporter.jasmineCli.test.ts delete mode 100644 packages/browserstack-service/tests/service.jasmineCliGates.test.ts diff --git a/packages/browserstack-service/tests/cli/cliUtils.test.ts b/packages/browserstack-service/tests/cli/cliUtils.test.ts index 5898bae7..99f23616 100644 --- a/packages/browserstack-service/tests/cli/cliUtils.test.ts +++ b/packages/browserstack-service/tests/cli/cliUtils.test.ts @@ -699,4 +699,14 @@ describe('CLIUtils', () => { expect(CLIUtils.getCliDir()).toBe('') }) }) + + describe('checkCLISupportedFrameworks', () => { + it.each(['mocha', 'cucumber', 'jasmine'])('routes %s to the CLI flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(true) + }) + + it.each([undefined, 'WebdriverIO-jasmine', 'Jasmine', 'unknown'])('keeps %s on the legacy flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(false) + }) + }) }) \ No newline at end of file diff --git a/packages/browserstack-service/tests/cli/index.test.ts b/packages/browserstack-service/tests/cli/index.test.ts index d4415b89..1e723f8b 100644 --- a/packages/browserstack-service/tests/cli/index.test.ts +++ b/packages/browserstack-service/tests/cli/index.test.ts @@ -3,6 +3,11 @@ import * as bstackLogger from '../../src/bstackLogger.js' import { BStackLogger } from '../../src/cli/cliLogger.js' import { BrowserstackCLI } from '../../src/cli/index.js' +import { CLIUtils } from '../../src/cli/cliUtils.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' const bstackLoggerSpy = vi.spyOn(bstackLogger.BStackLogger, 'logToFile') bstackLoggerSpy.mockImplementation(() => {}) @@ -119,4 +124,37 @@ describe('BrowserstackCLI bootstrap error surfacing', () => { expect(instance.config.apis).toEqual({ automate: { api: 'https://flat-only.example' } }) }) }) + + describe('setupTestFramework', () => { + afterEach(() => { + instance.testFramework = null + }) + + const setup = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + instance.setupTestFramework() + return instance.getTestFramework() + } + + it('constructs the jasmine framework for WebdriverIO-jasmine', () => { + const framework = setup('WebdriverIO-jasmine') + expect(framework).toBeInstanceOf(WdioJasmineTestFramework) + expect(framework).not.toBeInstanceOf(WdioMochaTestFramework) + expect(framework).toBeInstanceOf(TestFramework) + expect(framework!.getTestFrameworks()).toEqual(['WebdriverIO-jasmine']) + expect(framework!.getTestFrameworksVersions()).toEqual({ 'WebdriverIO-jasmine': '9.0.0' }) + }) + + it('still constructs the mocha framework for WebdriverIO-mocha', () => { + expect(setup('WebdriverIO-mocha')).toBeInstanceOf(WdioMochaTestFramework) + }) + + it('still constructs the cucumber framework for WebdriverIO-cucumber', () => { + expect(setup('WebdriverIO-cucumber')).toBeInstanceOf(WdioCucumberTestFramework) + }) + + it('leaves an unknown name unregistered', () => { + expect(setup('WebdriverIO-unknown')).toBeNull() + }) + }) }) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts deleted file mode 100644 index 424744d9..00000000 --- a/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { describe, expect, it, vi, afterEach } from 'vitest' -import * as bstackLogger from '../../src/bstackLogger.js' - -import { BrowserstackCLI } from '../../src/cli/index.js' -import { CLIUtils } from '../../src/cli/cliUtils.js' -import TestFramework from '../../src/cli/frameworks/testFramework.js' -import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' -import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' -import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -describe('CLIUtils.checkCLISupportedFrameworks', () => { - it.each(['mocha', 'cucumber', 'jasmine'])('routes %s to the CLI flow', (framework) => { - expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(true) - }) - - it.each([undefined, 'WebdriverIO-jasmine', 'Jasmine', 'unknown'])('keeps %s on the legacy flow', (framework) => { - expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(false) - }) -}) - -describe('BrowserstackCLI.setupTestFramework', () => { - const cli = BrowserstackCLI.getInstance() - - afterEach(() => { - vi.restoreAllMocks() - cli.testFramework = null - }) - - const setup = (name: string) => { - vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) - cli.setupTestFramework() - return cli.getTestFramework() - } - - it('constructs the jasmine framework for WebdriverIO-jasmine', () => { - const framework = setup('WebdriverIO-jasmine') - expect(framework).toBeInstanceOf(WdioJasmineTestFramework) - expect(framework).not.toBeInstanceOf(WdioMochaTestFramework) - expect(framework).toBeInstanceOf(TestFramework) - expect(framework!.getTestFrameworks()).toEqual(['WebdriverIO-jasmine']) - expect(framework!.getTestFrameworksVersions()).toEqual({ 'WebdriverIO-jasmine': '9.0.0' }) - }) - - it('still constructs the mocha framework for WebdriverIO-mocha', () => { - expect(setup('WebdriverIO-mocha')).toBeInstanceOf(WdioMochaTestFramework) - }) - - it('still constructs the cucumber framework for WebdriverIO-cucumber', () => { - expect(setup('WebdriverIO-cucumber')).toBeInstanceOf(WdioCucumberTestFramework) - }) - - it('leaves an unknown name unregistered', () => { - expect(setup('WebdriverIO-unknown')).toBeNull() - }) -}) diff --git a/packages/browserstack-service/tests/insights-handler.test.ts b/packages/browserstack-service/tests/insights-handler.test.ts index 7034c976..6d081093 100644 --- a/packages/browserstack-service/tests/insights-handler.test.ts +++ b/packages/browserstack-service/tests/insights-handler.test.ts @@ -10,6 +10,8 @@ import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' import { BrowserstackCLI } from '../src/cli/index.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' const log = logger('test') let insightsHandler: InsightsHandler @@ -736,6 +738,101 @@ describe('browserCommand', () => { insightsHandler.browserCommand('client:afterCommand', { sessionId: 's', method: 'm', endpoint: 'e', result: { value: 'random' } }, {}) expect(uploadEventDataSpy).toBeCalledTimes(0) }) + + describe('on the CLI flow', () => { + const cliBrowser = { on: vi.fn(), sessionId: 's', capabilities: {}, config: {}, execute: vi.fn() } as any + const command = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/title', body: {} } + const result = { ...command, result: { value: 'StackDemo' } } + const lastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any + let getInstanceSpy: ReturnType | undefined + + const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + } + + const handlerFor = (framework: string) => { + const handler = new InsightsHandler(cliBrowser, framework) + handler['getIdentifier'] = vi.fn().mockReturnValue('Nested outer outer passing test') + handler['_tests'] = { 'Nested outer outer passing test': { uuid: 'spec-uuid' } } + return handler + } + + beforeEach(() => { + delete process.env[TESTOPS_SCREENSHOT_ENV] + commandSpy.mockRestore() + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).not.toHaveBeenCalled() + expect(trackEvent).toHaveBeenCalledTimes(1) + const [state, hook, { logEntry }] = trackEvent.mock.calls[0] + expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) + expect(logEntry.kind).toBe('HTTP') + expect(logEntry.test_run_uuid).toBe('spec-uuid') + expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + }) + + it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).toHaveBeenCalledWith([{ + test_run_uuid: 'spec-uuid', + timestamp: expect.any(String), + kind: 'HTTP', + http_response: { path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } } + }]) + }) + + it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { + process.env[TESTOPS_SCREENSHOT_ENV] = 'true' + const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } + + const jasmineTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: jasmineTrack }) + await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) + getInstanceSpy!.mockRestore() + + const mochaTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: mochaTrack }) + await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) + + expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) + expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) + }) + + it('drops commands with no spec yet, as legacy did before the first spec', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + handler['getIdentifier'] = vi.fn().mockReturnValue(undefined) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, {} as any) + await handler.browserCommand('client:afterCommand', { ...result } as any, {} as any) + + expect(trackEvent).not.toHaveBeenCalled() + }) + }) }) describe('getIdentifier', () => { diff --git a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts deleted file mode 100644 index 76ea1a68..00000000 --- a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import path from 'node:path' -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' - -import TestReporter from '../src/reporter.js' -import { BrowserstackCLI } from '../src/cli/index.js' -import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' -import * as utils from '../src/util.js' -import * as bstackLogger from '../src/bstackLogger.js' - -vi.mock('uuid', () => ({ v4: () => 'legacy-uuid' })) -vi.mock('@wdio/reporter', () => import(path.join(process.cwd(), '__mocks__', '@wdio/reporter'))) -vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -const runnerConfig = { - type: 'runner', - cid: '0-0', - capabilities: { browserName: 'chrome', browserVersion: '151' }, - config: { framework: 'jasmine', hostname: 'hub.browserstack.com' }, - specs: ['/work/test/p2/nested.spec.js'], - sessionId: 'sessionId' -} - -const suite = { title: 'Nested outer', file: '/work/test/p2/nested.spec.js' } -const testStats = () => ({ - type: 'test', - uid: 'outer passing test0', - title: 'outer passing test', - fullTitle: 'Nested outer outer passing test', - start: new Date('2026-09-25T15:34:12.259Z'), - _duration: 0, - retries: 0, - state: 'pending', -}) -const hookStats = () => ({ type: 'hook', uid: 'h', title: '"before all" hook', start: new Date(), _duration: 0 }) -const logEntry = () => ({ timestamp: new Date().toISOString(), level: 'INFO', message: 'console line', kind: 'TEST_LOG' as const, http_response: {} }) - -describe('reporter jasmine arm', () => { - let reporter: TestReporter - let listener: Record> - let framework: WdioJasmineTestFramework - - const setCli = (running: boolean, testFramework: unknown) => { - vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ - isRunning: () => running, - getTestFramework: () => testFramework, - } as unknown as BrowserstackCLI) - } - - beforeEach(async () => { - vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') - vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) - framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') - for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { - vi.spyOn(framework, m) - } - vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') - vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterLog).mockReturnValue(undefined) - - reporter = new TestReporter({}) - await reporter.onRunnerStart(runnerConfig as any) - reporter.onSuiteStart(suite as any) - listener = { - testStarted: vi.spyOn(reporter['listener'], 'testStarted').mockImplementation(() => {}), - testFinished: vi.spyOn(reporter['listener'], 'testFinished').mockImplementation(() => {}), - hookStarted: vi.spyOn(reporter['listener'], 'hookStarted').mockImplementation(() => {}), - hookFinished: vi.spyOn(reporter['listener'], 'hookFinished').mockImplementation(() => {}), - logCreated: vi.spyOn(reporter['listener'], 'logCreated').mockImplementation(() => {}), - } - }) - - afterEach(() => { - vi.restoreAllMocks() - }) - - describe('on the CLI flow', () => { - beforeEach(() => setCli(true, framework)) - - it('feeds every test and hook event to the framework and enqueues nothing on the legacy Listener', async () => { - const stats = testStats() - await reporter.onHookStart(hookStats() as any) - await reporter.onHookEnd(hookStats() as any) - await reporter.onTestStart(stats as any) - await reporter.onTestEnd({ ...stats, state: 'passed' } as any) - await reporter.appendTestItemLog(logEntry()) - - expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterLog).toHaveBeenCalledTimes(1) - for (const spy of Object.values(listener)) { - expect(spy).not.toHaveBeenCalled() - } - }) - - it('passes the suite stack and suite file', async () => { - await reporter.onTestStart(testStats() as any) - expect(framework.onReporterTestStart).toHaveBeenCalledWith( - expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), - { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } - ) - }) - - it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { - await reporter.onTestStart(testStats() as any) - expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) - }) - - it('names the current test for Percy\'s testCase, as legacy getRunData did', async () => { - await reporter.onTestStart(testStats() as any) - expect((TestReporter as any).currentTest).toMatchObject({ uuid: 'cli-uuid', name: 'outer passing test' }) - }) - - it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { - const stats = testStats() as Record - await reporter.onTestEnd(stats as any) - expect(stats.end).toBeInstanceOf(Date) - const hook = hookStats() as Record - await reporter.onHookEnd(hook as any) - expect(hook.state).toBe('passed') - }) - - it('drops ', async () => { - await reporter.onTestStart({ ...testStats(), fullTitle: '' } as any) - await reporter.onTestEnd({ ...testStats(), fullTitle: '' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterTestEnd).not.toHaveBeenCalled() - }) - - it('sends nothing when Test Observability is opted out', async () => { - reporter['_observability'] = false - await reporter.onTestStart(testStats() as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(listener.testStarted).not.toHaveBeenCalled() - }) - - it('never falls back to the legacy Listener when the tracker is missing', async () => { - setCli(true, null) - await reporter.onTestStart(testStats() as any) - await reporter.appendTestItemLog(logEntry()) - expect(listener.testStarted).not.toHaveBeenCalled() - expect(listener.logCreated).not.toHaveBeenCalled() - }) - - it('reports unsynchronised while the framework has queued events', () => { - const spy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) - expect(reporter.isSynchronised).toBe(false) - spy.mockReturnValue(true) - expect(reporter.isSynchronised).toBe(true) - }) - }) - - describe('on the legacy flow', () => { - beforeEach(() => setCli(false, null)) - - it('enqueues on the legacy Listener and never touches the framework', async () => { - const stats = testStats() - await reporter.onHookStart(hookStats() as any) - await reporter.onHookEnd(hookStats() as any) - await reporter.onTestStart(stats as any) - await reporter.onTestEnd({ ...stats, state: 'passed' } as any) - reporter['_currentHook'] = {} - await reporter.appendTestItemLog(logEntry()) - - expect(listener.hookStarted).toHaveBeenCalledTimes(1) - expect(listener.hookFinished).toHaveBeenCalledTimes(1) - expect(listener.testStarted).toHaveBeenCalledTimes(1) - expect(listener.testFinished).toHaveBeenCalledTimes(1) - expect(listener.logCreated).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterLog).not.toHaveBeenCalled() - expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'legacy-uuid' }) - }) - }) - - describe('mocha on the CLI flow', () => { - beforeEach(() => setCli(true, framework)) - - it('leaves the mocha arm unchanged: no start/end publishing from the reporter', async () => { - reporter['_config']!.framework = 'mocha' - await reporter.onTestStart(testStats() as any) - await reporter.onTestEnd({ ...testStats(), state: 'passed' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(listener.testStarted).not.toHaveBeenCalled() - expect(listener.testFinished).not.toHaveBeenCalled() - }) - }) -}) diff --git a/packages/browserstack-service/tests/reporter.test.ts b/packages/browserstack-service/tests/reporter.test.ts index da59c2c2..2714f675 100644 --- a/packages/browserstack-service/tests/reporter.test.ts +++ b/packages/browserstack-service/tests/reporter.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi, beforeEach, afterEach, beforeAll, afterAll } import type { StdLog } from '../src/index.js' import TestReporter from '../src/reporter.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' @@ -354,4 +356,192 @@ describe('test-reporter', () => { expect(sendDataSpy).toBeCalledTimes(0) }) }) + + describe('jasmine CLI feed', () => { + const jasmineRunnerConfig = { + type: 'runner', + cid: '0-0', + capabilities: { browserName: 'chrome', browserVersion: '151' }, + config: { framework: 'jasmine', hostname: 'hub.browserstack.com' }, + specs: ['/work/test/p2/nested.spec.js'], + sessionId: 'sessionId' + } + const suite = { title: 'Nested outer', file: '/work/test/p2/nested.spec.js' } + const jasmineTestStats = () => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + _duration: 0, + retries: 0, + state: 'pending', + }) + const hookStats = () => ({ type: 'hook', uid: 'h', title: '"before all" hook', start: new Date(), _duration: 0 }) + const logEntry = () => ({ timestamp: new Date().toISOString(), level: 'INFO', message: 'console line', kind: 'TEST_LOG' as const, http_response: {} }) + + let reporter: TestReporter + let listener: Record> + let framework: WdioJasmineTestFramework + let getInstanceSpy: ReturnType | undefined + let getGitMetaDataSpy: ReturnType + let isIdleSpy: ReturnType | undefined + + const setCli = (running: boolean, testFramework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => running, + getTestFramework: () => testFramework, + } as unknown as BrowserstackCLI) + } + + beforeEach(async () => { + vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') + getGitMetaDataSpy = vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') + for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { + vi.spyOn(framework, m) + } + vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') + vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterLog).mockReturnValue(undefined) + + reporter = new TestReporter({}) + await reporter.onRunnerStart(jasmineRunnerConfig as any) + reporter.onSuiteStart(suite as any) + listener = { + testStarted: vi.spyOn(reporter['listener'], 'testStarted').mockImplementation(() => {}), + testFinished: vi.spyOn(reporter['listener'], 'testFinished').mockImplementation(() => {}), + hookStarted: vi.spyOn(reporter['listener'], 'hookStarted').mockImplementation(() => {}), + hookFinished: vi.spyOn(reporter['listener'], 'hookFinished').mockImplementation(() => {}), + logCreated: vi.spyOn(reporter['listener'], 'logCreated').mockImplementation(() => {}), + } + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + isIdleSpy?.mockRestore() + isIdleSpy = undefined + getGitMetaDataSpy.mockRestore() + for (const spy of Object.values(listener)) { + spy.mockRestore() + } + }) + + describe('on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('feeds every test and hook event to the framework and enqueues nothing on the legacy Listener', async () => { + const stats = jasmineTestStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + await reporter.appendTestItemLog(logEntry()) + + expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterLog).toHaveBeenCalledTimes(1) + for (const spy of Object.values(listener)) { + expect(spy).not.toHaveBeenCalled() + } + }) + + it('passes the suite stack and suite file', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect(framework.onReporterTestStart).toHaveBeenCalledWith( + expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), + { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } + ) + }) + + it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) + }) + + it('names the current test for Percy\'s testCase, as legacy getRunData did', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect((TestReporter as any).currentTest).toMatchObject({ uuid: 'cli-uuid', name: 'outer passing test' }) + }) + + it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { + const stats = jasmineTestStats() as Record + await reporter.onTestEnd(stats as any) + expect(stats.end).toBeInstanceOf(Date) + const hook = hookStats() as Record + await reporter.onHookEnd(hook as any) + expect(hook.state).toBe('passed') + }) + + it('drops ', async () => { + await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: '' } as any) + await reporter.onTestEnd({ ...jasmineTestStats(), fullTitle: '' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterTestEnd).not.toHaveBeenCalled() + }) + + it('sends nothing when Test Observability is opted out', async () => { + reporter['_observability'] = false + await reporter.onTestStart(jasmineTestStats() as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + }) + + it('never falls back to the legacy Listener when the tracker is missing', async () => { + setCli(true, null) + await reporter.onTestStart(jasmineTestStats() as any) + await reporter.appendTestItemLog(logEntry()) + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.logCreated).not.toHaveBeenCalled() + }) + + it('reports unsynchronised while the framework has queued events', () => { + isIdleSpy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) + expect(reporter.isSynchronised).toBe(false) + isIdleSpy.mockReturnValue(true) + expect(reporter.isSynchronised).toBe(true) + }) + }) + + describe('on the legacy flow', () => { + beforeEach(() => setCli(false, null)) + + it('enqueues on the legacy Listener and never touches the framework', async () => { + const stats = jasmineTestStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + reporter['_currentHook'] = {} + await reporter.appendTestItemLog(logEntry()) + + expect(listener.hookStarted).toHaveBeenCalledTimes(1) + expect(listener.hookFinished).toHaveBeenCalledTimes(1) + expect(listener.testStarted).toHaveBeenCalledTimes(1) + expect(listener.testFinished).toHaveBeenCalledTimes(1) + expect(listener.logCreated).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterLog).not.toHaveBeenCalled() + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: '123456789' }) + }) + }) + + describe('mocha on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('leaves the mocha arm unchanged: no start/end publishing from the reporter', async () => { + reporter['_config']!.framework = 'mocha' + await reporter.onTestStart(jasmineTestStats() as any) + await reporter.onTestEnd({ ...jasmineTestStats(), state: 'passed' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.testFinished).not.toHaveBeenCalled() + }) + }) + }) }) diff --git a/packages/browserstack-service/tests/service.jasmineCliGates.test.ts b/packages/browserstack-service/tests/service.jasmineCliGates.test.ts deleted file mode 100644 index 3a891c8f..00000000 --- a/packages/browserstack-service/tests/service.jasmineCliGates.test.ts +++ /dev/null @@ -1,254 +0,0 @@ -import path from 'node:path' - -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' - -import BrowserstackService from '../src/service.js' -import InsightsHandler from '../src/insights-handler.js' -import * as utils from '../src/util.js' -import * as skipReporter from '../src/cli/skipReporter.js' -import { BrowserstackCLI } from '../src/cli/index.js' -import * as bstackLogger from '../src/bstackLogger.js' -import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' -import { HookState } from '../src/cli/states/hookState.js' -import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' -import TestFramework from '../src/cli/frameworks/testFramework.js' - -vi.mock('fetch') -vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) -vi.mock('uuid', () => ({ v4: () => '123456789' })) -vi.mock('../src/data-store.js', () => ({ saveWorkerData: vi.fn() })) -vi.mock('../src/instrumentation/performance/performance-tester.js', () => ({ - default: { - start: vi.fn(), - end: vi.fn(), - startMonitoring: vi.fn(), - measureWrapper: vi.fn().mockImplementation((_name: string, fn: Function) => fn), - Measure: vi.fn().mockImplementation(() => (_target: any, _propertyKey: string, descriptor: PropertyDescriptor) => descriptor), - browser: undefined, - scenarioThatRan: [], - } -})) -vi.mock('../src/cli/index.js', () => ({ - BrowserstackCLI: { - getInstance: () => ({ - isRunning: () => false, - getTestFramework: () => null, - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - }) - } -})) - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -// What WDIO hands a jasmine hook: a copy of the last started spec (no `title`), then the hookName. -const lastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any -const mochaHook = (title: string) => ({ title, ctx: { test: { parent: { title: 'suite', tests: [], suites: [] } } } }) as any - -let getInstanceSpy: ReturnType | undefined - -const cliWith = (framework: unknown) => { - getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ - isRunning: () => true, - getTestFramework: () => framework, - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - } as any) -} - -const makeService = (framework: string) => new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) - -afterEach(() => { - getInstanceSpy?.mockRestore() - getInstanceSpy = undefined -}) - -describe('service hooks on the CLI flow (S4)', () => { - it('classifies jasmine hooks by hookName, without touching the missing title', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const getHookTypeSpy = vi.spyOn(utils, 'getHookType') - const service = makeService('jasmine') - - for (const hookName of ['beforeAll', 'beforeEach']) { - await service.beforeHook(lastSpec, {}, hookName) - } - for (const hookName of ['afterEach', 'afterAll']) { - await service.afterHook(lastSpec, {}, { passed: true } as any, hookName) - } - - expect(getHookTypeSpy).not.toHaveBeenCalled() - expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ - [TestFrameworkState.BEFORE_ALL, HookState.PRE], - [TestFrameworkState.BEFORE_EACH, HookState.PRE], - [TestFrameworkState.AFTER_EACH, HookState.POST], - [TestFrameworkState.AFTER_ALL, HookState.POST], - ]) - getHookTypeSpy.mockRestore() - }) - - it('sends nothing for a jasmine hook without a hookName, and does not throw', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const service = makeService('jasmine') - - await expect(service.beforeHook({} as any, {})).resolves.toBeUndefined() - await expect(service.afterHook({} as any, {}, { passed: true } as any)).resolves.toBeUndefined() - expect(trackEvent).not.toHaveBeenCalled() - }) - - it('never runs the mocha skip cascade for a failed jasmine hook, but still records the hook failure', async () => { - cliWith({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) - const service = makeService('jasmine') - - await service.afterHook({ ...lastSpec, ctx: { test: { parent: {} } } }, {}, { passed: false, error: new Error('beforeAll failed') } as any, 'beforeAll') - - expect(cascade).not.toHaveBeenCalled() - expect(service['_hookFailReasons']).toEqual(['beforeAll failed']) - cascade.mockRestore() - }) - - it('keeps mocha on its title classification and skip cascade', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) - const service = makeService('mocha') - - await service.beforeHook(mochaHook('"before each" hook for "t"'), {}, 'beforeEach') - await service.afterHook(mochaHook('"before all" hook for "t"'), {}, { passed: false, error: new Error('x') } as any, 'afterAll') - - expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ - [TestFrameworkState.BEFORE_EACH, HookState.PRE], - [TestFrameworkState.BEFORE_ALL, HookState.POST], - ]) - expect(cascade).toHaveBeenCalledTimes(1) - cascade.mockRestore() - }) -}) - -describe('service.beforeTest on the CLI flow (S5)', () => { - const runBeforeTest = async (framework: string, test: Record) => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const service = makeService(framework) - const annotate = vi.spyOn(service as any, '_setAnnotation').mockResolvedValue(undefined) - const getState = vi.spyOn(TestFramework, 'getState').mockReturnValue('spec-uuid') - await service.beforeTest(test as any) - getState.mockRestore() - return { annotate, trackEvent } - } - - it('annotates each jasmine spec with its full name, before the modules see TEST/PRE', async () => { - const { annotate, trackEvent } = await runBeforeTest('jasmine', lastSpec) - expect(annotate).toHaveBeenCalledWith('Test: Nested outer outer passing test') - const testPre = trackEvent.mock.calls.findIndex(([state]) => state === TestFrameworkState.TEST) - expect(annotate.mock.invocationCallOrder[0]).toBeLessThan(trackEvent.mock.invocationCallOrder[testPre]) - }) - - it('does not annotate mocha tests on the CLI flow', async () => { - const { annotate } = await runBeforeTest('mocha', { title: 't', parent: 'suite' }) - expect(annotate).not.toHaveBeenCalled() - }) -}) - -describe('service.before on the CLI flow (S6)', () => { - const registeredEvents = async (framework: string) => { - process.env.BROWSERSTACK_OBSERVABILITY = 'true' - cliWith(null) - const browser = { on: vi.fn(), sessionId: 's1', capabilities: {}, config: {}, execute: vi.fn(), executeScript: vi.fn() } as any - const service = new BrowserstackService({} as any, [{}] as any, { user: 'foo', key: 'bar', framework, capabilities: {} } as any) - await service.before(service['_config'] as any, [], browser) - delete process.env.BROWSERSTACK_OBSERVABILITY - return vi.mocked(browser.on).mock.calls.map(([event]: [string]) => event) - } - - it('registers command and result for jasmine', async () => { - const events = await registeredEvents('jasmine') - expect(events).toContain('command') - expect(events).toContain('result') - }) - - it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { - const events = await registeredEvents(framework) - expect(events).toContain('result') - expect(events).not.toContain('command') - }) -}) - -describe('insights-handler.browserCommand on the CLI flow (S6)', () => { - const browser = { on: vi.fn(), sessionId: 's', capabilities: {}, config: {}, execute: vi.fn() } as any - const command = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/title', body: {} } - const result = { ...command, result: { value: 'StackDemo' } } - - const handlerFor = (framework: string) => { - const handler = new InsightsHandler(browser, framework) - handler['getIdentifier'] = vi.fn().mockReturnValue('Nested outer outer passing test') - handler['_tests'] = { 'Nested outer outer passing test': { uuid: 'spec-uuid' } } - return handler - } - - beforeEach(() => { - delete process.env[TESTOPS_SCREENSHOT_ENV] - }) - - it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const handler = handlerFor('jasmine') - const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) - - await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) - await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) - - expect(logCreated).not.toHaveBeenCalled() - expect(trackEvent).toHaveBeenCalledTimes(1) - const [state, hook, { logEntry }] = trackEvent.mock.calls[0] - expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) - expect(logEntry.kind).toBe('HTTP') - expect(logEntry.test_run_uuid).toBe('spec-uuid') - expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) - }) - - it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { - const handler = handlerFor('jasmine') - const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) - - await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) - await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) - - expect(logCreated).toHaveBeenCalledWith([{ - test_run_uuid: 'spec-uuid', - timestamp: expect.any(String), - kind: 'HTTP', - http_response: { path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } } - }]) - }) - - it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { - process.env[TESTOPS_SCREENSHOT_ENV] = 'true' - const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } - - const jasmineTrack = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent: jasmineTrack }) - await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) - getInstanceSpy!.mockRestore() - - const mochaTrack = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent: mochaTrack }) - await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) - - expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) - expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) - }) - - it('drops commands with no spec yet, as legacy did before the first spec', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const handler = handlerFor('jasmine') - handler['getIdentifier'] = vi.fn().mockReturnValue(undefined) - - await handler.browserCommand('client:beforeCommand', { ...command } as any, {} as any) - await handler.browserCommand('client:afterCommand', { ...result } as any, {} as any) - - expect(trackEvent).not.toHaveBeenCalled() - }) -}) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index aa32efbc..15907aa6 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -7,6 +7,8 @@ import BrowserstackService from '../src/service.js' import * as utils from '../src/util.js' import InsightsHandler from '../src/insights-handler.js' import { BrowserstackCLI } from '../src/cli/index.js' +import * as skipReporter from '../src/cli/skipReporter.js' +import TestFramework from '../src/cli/frameworks/testFramework.js' import AccessibilityModule from '../src/cli/modules/accessibilityModule.js' import * as bstackLogger from '../src/bstackLogger.js' import AutomationFramework from '../src/cli/frameworks/automationFramework.js' @@ -20,6 +22,8 @@ const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' const sessionBaseUrl = 'https://api.browserstack.com/automate/sessions' const sessionId = 'session123' const sessionIdA = 'session456' +// What WDIO hands a jasmine hook: a copy of the last started spec (no `title`), then the hookName. +const jasmineLastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any vi.mock('fetch') vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) @@ -850,6 +854,41 @@ describe('before', () => { expect(browserA.overwriteCommand).not.toHaveBeenCalled() expect(browserB.overwriteCommand).toHaveBeenCalledWith('execute', expect.any(Function)) }) + + describe('command and result registration on the CLI flow', () => { + let getInstanceSpy: ReturnType | undefined + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + const registeredEvents = async (framework: string) => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + const cliBrowser = { on: vi.fn(), sessionId: 's1', capabilities: {}, config: {}, execute: vi.fn(), executeScript: vi.fn() } as any + const service = new BrowserstackService({} as any, [{}] as any, { user: 'foo', key: 'bar', framework, capabilities: {} } as any) + await service.before(service['_config'] as any, [], cliBrowser) + delete process.env.BROWSERSTACK_OBSERVABILITY + return vi.mocked(cliBrowser.on).mock.calls.map(([event]: [string]) => event) + } + + it('registers command and result for jasmine', async () => { + const events = await registeredEvents('jasmine') + expect(events).toContain('command') + expect(events).toContain('result') + }) + + it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { + const events = await registeredEvents(framework) + expect(events).toContain('result') + expect(events).not.toContain('command') + }) + }) }) describe('beforeHook', () => { @@ -1154,6 +1193,42 @@ describe('beforeTest', () => { ) }) }) + + describe('on the CLI flow', () => { + let getInstanceSpy: ReturnType | undefined + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + const runBeforeTest = async (framework: string, test: Record) => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => ({ trackEvent }), + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + const service = new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + const annotate = vi.spyOn(service as any, '_setAnnotation').mockResolvedValue(undefined) + const getState = vi.spyOn(TestFramework, 'getState').mockReturnValue('spec-uuid') + await service.beforeTest(test as any) + getState.mockRestore() + return { annotate, trackEvent } + } + + it('annotates each jasmine spec with its full name, before the modules see TEST/PRE', async () => { + const { annotate, trackEvent } = await runBeforeTest('jasmine', jasmineLastSpec) + expect(annotate).toHaveBeenCalledWith('Test: Nested outer outer passing test') + const testPre = trackEvent.mock.calls.findIndex(([state]) => state === TestFrameworkState.TEST) + expect(annotate.mock.invocationCallOrder[0]).toBeLessThan(trackEvent.mock.invocationCallOrder[testPre]) + }) + + it('does not annotate mocha tests on the CLI flow', async () => { + const { annotate } = await runBeforeTest('mocha', { title: 't', parent: 'suite' }) + expect(annotate).not.toHaveBeenCalled() + }) + }) }) describe('afterTest', () => { @@ -2638,6 +2713,80 @@ describe('beforeHook (CLI hook reporting)', () => { expect(service['_insightsHandler']!.beforeHook).toHaveBeenCalledTimes(1) }) + + describe('classification by framework', () => { + const mochaHook = (title: string) => ({ title, ctx: { test: { parent: { title: 'suite', tests: [], suites: [] } } } }) as any + const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + } + const makeService = (framework: string) => new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + + it('classifies jasmine hooks by hookName, without touching the missing title', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const getHookTypeSpy = vi.spyOn(utils, 'getHookType') + const service = makeService('jasmine') + + for (const hookName of ['beforeAll', 'beforeEach']) { + await service.beforeHook(jasmineLastSpec, {}, hookName) + } + for (const hookName of ['afterEach', 'afterAll']) { + await service.afterHook(jasmineLastSpec, {}, { passed: true } as any, hookName) + } + + expect(getHookTypeSpy).not.toHaveBeenCalled() + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.AFTER_EACH, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + getHookTypeSpy.mockRestore() + }) + + it('sends nothing for a jasmine hook without a hookName, and does not throw', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const service = makeService('jasmine') + + await expect(service.beforeHook({} as any, {})).resolves.toBeUndefined() + await expect(service.afterHook({} as any, {}, { passed: true } as any)).resolves.toBeUndefined() + expect(trackEvent).not.toHaveBeenCalled() + }) + + it('never runs the mocha skip cascade for a failed jasmine hook, but still records the hook failure', async () => { + cliWith({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('jasmine') + + await service.afterHook({ ...jasmineLastSpec, ctx: { test: { parent: {} } } }, {}, { passed: false, error: new Error('beforeAll failed') } as any, 'beforeAll') + + expect(cascade).not.toHaveBeenCalled() + expect(service['_hookFailReasons']).toEqual(['beforeAll failed']) + cascade.mockRestore() + }) + + it('keeps mocha on its title classification and skip cascade', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('mocha') + + await service.beforeHook(mochaHook('"before each" hook for "t"'), {}, 'beforeEach') + await service.afterHook(mochaHook('"before all" hook for "t"'), {}, { passed: false, error: new Error('x') } as any, 'afterAll') + + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + ]) + expect(cascade).toHaveBeenCalledTimes(1) + cascade.mockRestore() + }) + }) }) describe('_isAppAutomate honors skipAppOverride', () => { From 05be96ac38a8f0001271f9c683ee5bbf16c23dc6 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 7 Oct 2026 11:37:16 +0530 Subject: [PATCH 14/15] refactor(cli): give the jasmine framework one entry point and move its session verdict to AutomateModule (SDK-7772) The reporter now feeds WdioJasmineTestFramework through trackEvent with source 'reporter'; reporter events are applied before trackEvent awaits, and TEST PRE hands the minted uuid back on args.testUuid. TestHubModule is wired once in loadModules instead of on every reporter event. service.after() passes legacy's session verdict inputs (runner result, specsRan and the three reason lists) on EXECUTE POST for jasmine only; AutomateModule computes the verdict from them, so the framework class no longer keeps verdict state. The pending-event counter and isIdle() are per instance. Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 171 ++++--------- .../browserstack-service/src/cli/index.ts | 6 +- .../src/cli/modules/automateModule.ts | 58 ++++- packages/browserstack-service/src/reporter.ts | 26 +- packages/browserstack-service/src/service.ts | 13 +- .../tests/cli/index.test.ts | 37 +++ .../tests/cli/modules/automateModule.test.ts | 107 ++++++--- .../cli/wdioJasmineTestFramework.test.ts | 226 ++++++++---------- .../tests/reporter.test.ts | 88 ++++--- .../tests/service.test.ts | 63 +++++ 10 files changed, 456 insertions(+), 339 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 7b946760..5b983ebe 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -5,8 +5,6 @@ import type { Frameworks } from '@wdio/types' import type { HookStats, TestStats } from '@wdio/reporter' import TestFramework from './testFramework.js' -import AutomationFramework from './automationFramework.js' -import { AutomationFrameworkConstants } from './constants/automationFrameworkConstants.js' import { TestFrameworkState } from '../states/testFrameworkState.js' import { HookState } from '../states/hookState.js' import TestFrameworkInstance from '../instances/testFrameworkInstance.js' @@ -25,17 +23,6 @@ export interface JasmineSuiteContext { suiteFile?: string } -/** The Automate verdict legacy `service.after()` marks a session with. */ -export interface SessionVerdict { - status: 'passed' | 'failed' - reason?: string -} - -interface SessionFailure { - reason: string - fromHook: boolean -} - interface FailureFields { failure?: Array<{ backtrace: string[] }> failureReason?: string @@ -48,10 +35,11 @@ interface FailureFields { * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), * hook taxonomy and skip/exclude states differ from mocha's. * - * Two event sources, as on the legacy flow: - * - the WDIO reporter (`reporter.ts`) feeds Test Observability only — it is the one jasmine source that - * sees pending, focused-out and beforeAll-failed specs. Its hooks are not awaited by WDIO, so its events - * are applied through one ordered queue and sent straight to TestHubModule, never to the other modules; + * Two event sources, as on the legacy flow, both through `trackEvent`: + * - the WDIO reporter (`reporter.ts`, `source: 'reporter'`) feeds Test Observability only — it is the one + * jasmine source that sees pending, focused-out and beforeAll-failed specs. Its hooks are not awaited by + * WDIO, so its events are applied through one ordered queue and sent straight to TestHubModule, never to + * the other modules; * - the service's awaited beforeTest/afterTest drive the product modules (a11y, Automate, Percy) through * the observers, for executed specs only, with TestHub skipped. */ @@ -70,13 +58,6 @@ export default class WdioJasmineTestFramework extends TestFramework { static KEY_HOOK_FAILURE_TYPE = 'hook_failure_type' static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' - static #pendingEvents = 0 - static #serviceHookStates = new Set([ - TestFrameworkState.BEFORE_ALL, - TestFrameworkState.BEFORE_EACH, - TestFrameworkState.AFTER_EACH, - TestFrameworkState.AFTER_ALL, - ]) static #hookTypes = new Map([ ['beforeAll', 'BEFORE_ALL'], ['afterAll', 'AFTER_ALL'], @@ -97,23 +78,16 @@ export default class WdioJasmineTestFramework extends TestFramework { #openHook: TestFrameworkInstance | null = null #lastSpec: TestFrameworkInstance | null = null #queue: Promise = Promise.resolve() - // Legacy `service.ts` session verdict inputs: `_specsRan` and the runner result are per worker, - // the reason lists per session (`onReload` resets them). - #specsRan = false - #runnerFailedCount = 0 - // In the order they happened, as legacy pushed them - #sessionFailures = new Map() - static #current: WdioJasmineTestFramework | null = null + #pendingEvents = 0 constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { super(testFrameworks, testFrameworkVersions, binSessionId) - WdioJasmineTestFramework.#current = this logger.debug('WdioJasmineTestFramework: constructed') } /** True when no reporter event is still waiting to be dispatched (the reporter's `isSynchronised`). */ - static isIdle() { - return WdioJasmineTestFramework.#pendingEvents === 0 + isIdle() { + return this.#pendingEvents === 0 } /** The WDIO `hookName` a jasmine hook runs under, as the hook-type key `getHookType` returns for mocha titles. */ @@ -121,27 +95,32 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#hookTypes.get(hookName ?? '') ?? 'unknown' } + /** The state the reporter reports a hook in: BEFORE_ALL/AFTER_ALL, else NONE (each-hooks are not reported). */ + static reporterHookState(hookTitle: string | undefined): State { + return WdioJasmineTestFramework.#hookState(getHookType(String(hookTitle ?? '').toLowerCase())) ?? TestFrameworkState.NONE + } + setTestHubModule(testHub: TestHubModule | null | undefined) { this.#testHub = testHub ?? null } /** - * The service's hooks call this. INIT_TEST pins the spec's instance and uuid; TEST PRE/POST reach the - * product modules. LOG_REPORT carries nothing jasmine needs: the result comes from the reporter. + * Reporter events (`source: 'reporter'`) are applied before this returns its promise: TEST PRE writes the + * spec's uuid to `args.testUuid`. The service's hooks call this without a source: INIT_TEST pins the + * spec's instance and uuid; TEST PRE/POST reach the product modules. LOG_REPORT and the hook states carry + * nothing jasmine needs: results come from the reporter. */ async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + if (args.source === 'reporter') { + this.#trackReporterEvent(testFrameworkState, hookState, args) + return + } if (testFrameworkState === TestFrameworkState.LOG) { - this.onReporterLog(args.logEntry as Record) + this.#log(args.logEntry as Record) return } try { await this.#queue - if (WdioJasmineTestFramework.#serviceHookStates.has(testFrameworkState)) { - if (hookState === HookState.POST) { - this.#recordHookResult(args.result as Frameworks.TestResult | undefined) - } - return - } if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { return } @@ -157,7 +136,6 @@ export default class WdioJasmineTestFramework extends TestFramework { if (hookState === HookState.PRE) { this.#suiteTitles.set(instance, args.suiteTitle) } else { - this.#recordTestResult(args.result as Frameworks.TestResult | undefined) // afterTest hands over the raw `Jasmine__TopLevel__Suite`; the modules need the describe chain beforeTest derived moduleArgs.suiteTitle = this.#suiteTitles.get(instance) ?? args.suiteTitle this.#suiteTitles.delete(instance) @@ -171,7 +149,24 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterTestStart(testStats: TestStats, context: JasmineSuiteContext) { + #trackReporterEvent(testFrameworkState: State, hookState: State, args: Record) { + const context = args.context as JasmineSuiteContext + if (testFrameworkState === TestFrameworkState.TEST) { + if (hookState === HookState.PRE) { + args.testUuid = this.#testStarted(args.testStats as TestStats, context) + } else { + this.#testEnded(args.testStats as TestStats, context) + } + } else if (testFrameworkState === TestFrameworkState.LOG) { + this.#log(args.logEntry as Record | undefined) + } else if (hookState === HookState.PRE) { + this.#hookStarted(testFrameworkState, args.hookStats as HookStats, context) + } else { + this.#hookEnded(testFrameworkState, args.hookStats as HookStats) + } + } + + #testStarted(testStats: TestStats, context: JasmineSuiteContext) { try { logger.debug(`WdioJasmineTestFramework: test start uid=${testStats.uid}`) const instance = this.#createInstance() @@ -207,7 +202,7 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterTestEnd(testStats: TestStats, context: JasmineSuiteContext) { + #testEnded(testStats: TestStats, context: JasmineSuiteContext) { try { const instance = this.#specInstances.get(testStats.uid) if (!instance) { @@ -221,9 +216,6 @@ export default class WdioJasmineTestFramework extends TestFramework { const state = testStats.state const error = testStats.error - if (state === 'failed') { - this.#runnerFailedCount++ - } let result: string = state let failure: FailureFields = {} if (state === 'failed') { @@ -261,12 +253,11 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterHookStart(hookStats: HookStats, context: JasmineSuiteContext) { + #hookStarted(hookFrameworkState: State, hookStats: HookStats, context: JasmineSuiteContext) { try { const title = hookStats.title const key = getHookType(String(title ?? '').toLowerCase()) - const hookFrameworkState = WdioJasmineTestFramework.#hookState(key) - if (!hookFrameworkState || context.scopes.length === 0) { + if (!WdioJasmineTestFramework.#hookState(key) || context.scopes.length === 0) { logger.debug(`WdioJasmineTestFramework: hook start not reported title=${title} scopes=${context.scopes.length}`) return } @@ -306,7 +297,7 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterHookEnd(hookStats: HookStats) { + #hookEnded(hookFrameworkState: State, hookStats: HookStats) { try { const instance = this.#hookInstances.get(hookStats.uid) if (!instance) { @@ -314,7 +305,6 @@ export default class WdioJasmineTestFramework extends TestFramework { } this.#hookInstances.delete(hookStats.uid) const key = getHookType(String(hookStats.title ?? '').toLowerCase()) - const hookFrameworkState = WdioJasmineTestFramework.#hookState(key)! const state = hookStats.state const error = hookStats.error @@ -364,7 +354,7 @@ export default class WdioJasmineTestFramework extends TestFramework { * Console logs: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. * HTTP command logs and screenshots name their spec (`test_run_uuid`), as legacy did; unknown uuids are dropped. */ - onReporterLog(logEntry: Record | undefined) { + #log(logEntry: Record | undefined) { try { if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { return @@ -395,73 +385,6 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - /** - * The verdict legacy `service.after()` would mark the live session with: passed only when the runner - * reported no failed spec, a spec ran, and no test or hook failed, with `ignoreHooksStatus` handled as - * coded there. `null` for any other session: legacy marked a reloaded session once, in `onReload`. - * `undefined` when this worker does not run jasmine on the CLI flow. - */ - static sessionVerdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null | undefined { - const current = WdioJasmineTestFramework.#current - return current ? current.#verdict(sessionId, ignoreHooksStatus) : undefined - } - - #verdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null { - if (!sessionId || sessionId !== WdioJasmineTestFramework.#liveSessionId()) { - return null - } - const failures = this.#sessionFailures.get(sessionId) ?? [] - const pureTestFailReasons = failures.filter(f => !f.fromHook).map(f => f.reason) - const hookFailReasons = failures.filter(f => f.fromHook).map(f => f.reason) - const failReasons = ignoreHooksStatus ? pureTestFailReasons : failures.map(f => f.reason) - const joined = (reasons: string[]) => reasons.length > 0 ? reasons.join('\n') : undefined - - if (this.#runnerFailedCount === 0 && this.#specsRan) { - const reasons = ignoreHooksStatus ? pureTestFailReasons : failReasons - return reasons.length > 0 ? { status: 'failed', reason: joined(reasons) } : { status: 'passed' } - } - if (ignoreHooksStatus && this.#specsRan) { - const hasOnlyHookFailures = failReasons.length === 0 && hookFailReasons.length > 0 - if (hasOnlyHookFailures && pureTestFailReasons.length === 0) { - return { status: 'passed' } - } - return { status: 'failed', reason: joined(pureTestFailReasons) } - } - return { - status: 'failed', - reason: ignoreHooksStatus && pureTestFailReasons.length > 0 ? joined(pureTestFailReasons) : joined(failReasons), - } - } - - /** Legacy `service.afterTest()`: the spec ran; a failure counts toward the session. */ - #recordTestResult(result: Frameworks.TestResult | undefined) { - this.#specsRan = true - if (!result || result.passed || result.skipped) { - return - } - this.#recordFailure((result.error && result.error.message) || 'Unknown Error', false) - } - - /** Legacy `service.afterHook()`, any of the four hook types. */ - #recordHookResult(result: Frameworks.TestResult | undefined) { - if (!result || result.passed) { - return - } - this.#recordFailure((result.error && result.error.message) || 'Hook failed', true) - } - - #recordFailure(reason: string, fromHook: boolean) { - const sessionId = WdioJasmineTestFramework.#liveSessionId() - const failures = this.#sessionFailures.get(sessionId) ?? [] - failures.push({ reason, fromHook }) - this.#sessionFailures.set(sessionId, failures) - } - - static #liveSessionId(): string { - const autoInstance = AutomationFramework.getTrackedInstance() - return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' - } - #createInstance() { const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) const instance = new TestFrameworkInstance( @@ -525,11 +448,11 @@ export default class WdioJasmineTestFramework extends TestFramework { } #enqueue(label: string, step: () => Promise) { - WdioJasmineTestFramework.#pendingEvents++ + this.#pendingEvents++ this.#queue = this.#queue.then(step).catch((error) => { logger.error(`WdioJasmineTestFramework: ${label} failed: ${util.format(error)}`) }).finally(() => { - WdioJasmineTestFramework.#pendingEvents-- + this.#pendingEvents-- }) } diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 735ccc6d..56eaf74c 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -179,7 +179,11 @@ export class BrowserstackCLI { this.modules[ObservabilityModule.MODULE_NAME] = new ObservabilityModule(startBinResponse.observability) } - this.modules[TestHubModule.MODULE_NAME] = new TestHubModule(startBinResponse.testhub) + const testHubModule = new TestHubModule(startBinResponse.testhub) + this.modules[TestHubModule.MODULE_NAME] = testHubModule + if (this.testFramework instanceof WdioJasmineTestFramework) { + this.testFramework.setTestHubModule(testHubModule) + } // Custom-tag (multi Test-Case-ID) tagging rides the per-test event_json // to TestHub, so it is gated on the testhub pipeline being active. diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index c6554f50..666ce064 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -1,5 +1,4 @@ import BaseModule from './baseModule.js' -import WdioJasmineTestFramework from '../frameworks/wdioJasmineTestFramework.js' import { BrowserstackCLI } from '../index.js' import { BStackLogger } from '../cliLogger.js' import TestFramework from '../frameworks/testFramework.js' @@ -25,6 +24,20 @@ interface TestResult { reason?: string } +/** What legacy `service.after()` computed the jasmine session verdict from, as the service tracked it. */ +interface SessionVerdictInputs { + result: number + specsRan: boolean + failReasons: string[] + pureTestFailReasons: string[] + hookFailReasons: string[] +} + +interface SessionVerdict { + status: 'passed' | 'failed' + reason?: string +} + interface SessionData { lastTestName: string appliedName?: string // last name successfully PUT for this session, for de-duping @@ -361,19 +374,18 @@ export default class AutomateModule extends BaseModule { } } - async onAfterExecute() { + async onAfterExecute(args: { sessionVerdictInputs?: SessionVerdictInputs } = {}) { this.logger.debug('onAfterExecute: inside automate module after execute hook!') const userName = this.config.userName as string const accessKey = this.config.accessKey as string const testContextOptions = this.config.testContextOptions as TestContextOptions - // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options - const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } - const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true - const liveSessionId = this.liveSessionId() - if (WdioJasmineTestFramework.sessionVerdict(liveSessionId, ignoreHooksStatus) !== undefined) { - await this.markJasmineSessions(liveSessionId, ignoreHooksStatus) + if (args?.sessionVerdictInputs) { + // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options + const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } + const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true + await this.markJasmineSessions(this.liveSessionId(), this.jasmineVerdict(args.sessionVerdictInputs, ignoreHooksStatus)) this.sessionMap.clear() return } @@ -424,12 +436,37 @@ export default class AutomateModule extends BaseModule { return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' } + /** + * Legacy `service.after()`'s session status: passed only when the runner reported no failed spec, a spec + * ran, and no test or hook failed, with `ignoreHooksStatus` handled as coded there. + */ + private jasmineVerdict(inputs: SessionVerdictInputs, ignoreHooksStatus: boolean): SessionVerdict { + const { result, specsRan, failReasons, pureTestFailReasons, hookFailReasons } = inputs + const joined = (reasons: string[]) => reasons.length > 0 ? reasons.join('\n') : undefined + + if (result === 0 && specsRan) { + const reasons = ignoreHooksStatus ? pureTestFailReasons : failReasons + return reasons.length > 0 ? { status: 'failed', reason: joined(reasons) } : { status: 'passed' } + } + if (ignoreHooksStatus && specsRan) { + const hasOnlyHookFailures = failReasons.length === 0 && hookFailReasons.length > 0 + if (hasOnlyHookFailures && pureTestFailReasons.length === 0) { + return { status: 'passed' } + } + return { status: 'failed', reason: joined(pureTestFailReasons) } + } + return { + status: 'failed', + reason: ignoreHooksStatus && pureTestFailReasons.length > 0 ? joined(pureTestFailReasons) : joined(failReasons), + } + } + /** * Jasmine: legacy `service.after()` marked only the live session, with the worker's status, the last * name, and the test and hook failure reasons. A session nothing registered (a beforeAll failed before * any spec ran) is still marked, and a reloaded one is left to the mark `onReload` already sent. */ - private async markJasmineSessions(liveSessionId: string, ignoreHooksStatus: boolean) { + private async markJasmineSessions(liveSessionId: string, verdict: SessionVerdict) { const testContextOptions = this.config.testContextOptions as TestContextOptions const auth = { user: this.config.userName as string, key: this.config.accessKey as string } const sessionIds = new Set(this.sessionMap.keys()) @@ -440,8 +477,7 @@ export default class AutomateModule extends BaseModule { for (const sessionId of sessionIds) { try { await this.flushSessionName(sessionId) - const verdict = WdioJasmineTestFramework.sessionVerdict(sessionId, ignoreHooksStatus) - if (!verdict || testContextOptions.skipSessionStatus) { + if (!liveSessionId || sessionId !== liveSessionId || testContextOptions.skipSessionStatus) { continue } const name = testContextOptions.skipSessionName ? undefined : this.sessionMap.get(sessionId)?.lastTestName || undefined diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index c4d839e3..1345c8b5 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -7,7 +7,8 @@ import { BrowserstackCLI } from './cli/index.js' import { reportSkippedTest, resolveSpecFile } from './cli/skipReporter.js' import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' import type { JasmineSuiteContext } from './cli/frameworks/wdioJasmineTestFramework.js' -import type TestHubModule from './cli/modules/testHubModule.js' +import { TestFrameworkState } from './cli/states/testFrameworkState.js' +import { HookState } from './cli/states/hookState.js' import * as url from 'node:url' import { v4 as uuidv4 } from 'uuid' @@ -76,7 +77,7 @@ class _TestReporter extends WDIOReporter { public async appendTestItemLog(stdLog: StdLog) { if (this.isCliJasmine()) { - this.cliJasmineFramework()?.onReporterLog(stdLog as unknown as Record) + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.LOG, HookState.POST, { source: 'reporter', logEntry: stdLog }) return } if (this._currentHook.uuid && !this._currentHook.finished) { @@ -146,7 +147,7 @@ class _TestReporter extends WDIOReporter { * asynchronously, so hold the worker until it has drained. Always true on every other path. */ get isSynchronised() { - return WdioJasmineTestFramework.isIdle() + return !this.isCliJasmine() || (this.cliJasmineFramework()?.isIdle() ?? true) } /** On the CLI flow jasmine's test/hook/log events feed the framework tracker, never the legacy Listener. */ @@ -155,13 +156,8 @@ class _TestReporter extends WDIOReporter { } cliJasmineFramework() { - const cli = BrowserstackCLI.getInstance() - const framework = cli.getTestFramework() - if (!(framework instanceof WdioJasmineTestFramework)) { - return null - } - framework.setTestHubModule(cli.modules?.TestHubModule as TestHubModule | undefined) - return framework + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioJasmineTestFramework ? framework : null } jasmineSuiteContext(): JasmineSuiteContext { @@ -194,7 +190,7 @@ class _TestReporter extends WDIOReporter { testStats.end ||= new Date() if (this.isCliJasmine()) { - this.cliJasmineFramework()?.onReporterTestEnd(testStats, this.jasmineSuiteContext()) + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.POST, { source: 'reporter', testStats, context: this.jasmineSuiteContext() }) return } this.listener.testFinished(await this.getRunData(testStats, 'TestRunFinished')) @@ -208,7 +204,9 @@ class _TestReporter extends WDIOReporter { return } if (this.isCliJasmine()) { - const cliUuid = this.cliJasmineFramework()?.onReporterTestStart(testStats, this.jasmineSuiteContext()) + const args: Record = { source: 'reporter', testStats, context: this.jasmineSuiteContext() } + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + const cliUuid = args.testUuid as string | undefined if (cliUuid) { _TestReporter.currentTest.uuid = cliUuid _TestReporter.currentTest.name = testStats.title @@ -230,7 +228,7 @@ class _TestReporter extends WDIOReporter { return } if (this.isCliJasmine()) { - this.cliJasmineFramework()?.onReporterHookStart(hookStats, this.jasmineSuiteContext()) + this.cliJasmineFramework()?.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), HookState.PRE, { source: 'reporter', hookStats, context: this.jasmineSuiteContext() }) return } @@ -252,7 +250,7 @@ class _TestReporter extends WDIOReporter { if (!hookStats.state && !hookStats.error) { hookStats.state = 'passed' } - this.cliJasmineFramework()?.onReporterHookEnd(hookStats) + this.cliJasmineFramework()?.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), HookState.POST, { source: 'reporter', hookStats, context: this.jasmineSuiteContext() }) return } const identifier = this.getHookIdentifier(hookStats) diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 104a1a5d..4a24bd1e 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -794,7 +794,18 @@ export default class BrowserstackService implements Services.ServiceInstance { } catch (flushErr) { BStackLogger.debug(`Exception flushing deferred test finish in after(): ${util.format(flushErr)}`) } - await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.EXECUTE, HookState.POST, {}) + const executeArgs = this._config.framework === 'jasmine' + ? { + sessionVerdictInputs: { + result, + specsRan: this._specsRan, + failReasons: [...this._failReasons], + pureTestFailReasons: [...this._pureTestFailReasons], + hookFailReasons: [...this._hookFailReasons], + } + } + : {} + await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.EXECUTE, HookState.POST, executeArgs) } // if (setSessionStatus) { diff --git a/packages/browserstack-service/tests/cli/index.test.ts b/packages/browserstack-service/tests/cli/index.test.ts index 1e723f8b..6c8bf92c 100644 --- a/packages/browserstack-service/tests/cli/index.test.ts +++ b/packages/browserstack-service/tests/cli/index.test.ts @@ -4,6 +4,7 @@ import { BStackLogger } from '../../src/cli/cliLogger.js' import { BrowserstackCLI } from '../../src/cli/index.js' import { CLIUtils } from '../../src/cli/cliUtils.js' +import APIUtils from '../../src/cli/apiUtils.js' import TestFramework from '../../src/cli/frameworks/testFramework.js' import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' @@ -157,4 +158,40 @@ describe('BrowserstackCLI bootstrap error surfacing', () => { expect(setup('WebdriverIO-unknown')).toBeNull() }) }) + + describe('loadModules TestHub wiring', () => { + const savedEnv = { ...process.env } + + afterEach(() => { + instance.testFramework = null + instance.modulesLoaded = false + process.env = { ...savedEnv } + }) + + const load = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + vi.spyOn(APIUtils, 'updateURLSForGRR').mockImplementation(() => {}) + vi.spyOn(CLIUtils, 'getAutomationFrameworkDetail').mockReturnValue({ name: 'WebdriverIO', version: { WebdriverIO: '9.0.0' } }) + instance.modulesLoaded = false + instance.loadModules({ + binSessionId: 'b1', + config: '{}', + testhub: { jwt: 'jwt', buildHashedId: 'build' } + } as any) + } + + it('hands the jasmine framework its TestHubModule once, when the module is built', () => { + const setTestHubModule = vi.spyOn(WdioJasmineTestFramework.prototype, 'setTestHubModule') + load('WebdriverIO-jasmine') + expect(setTestHubModule).toHaveBeenCalledTimes(1) + expect(setTestHubModule.mock.calls[0][0]).toBe(instance.modules.TestHubModule) + }) + + it('leaves the mocha framework untouched', () => { + const setTestHubModule = vi.spyOn(WdioJasmineTestFramework.prototype, 'setTestHubModule') + load('WebdriverIO-mocha') + expect(instance.modules.TestHubModule).toBeDefined() + expect(setTestHubModule).not.toHaveBeenCalled() + }) + }) }) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 4d1fbf45..3d75673f 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -9,7 +9,6 @@ import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/te import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' -import WdioJasmineTestFramework from '../../../src/cli/frameworks/wdioJasmineTestFramework.js' import { BrowserstackCLI } from '../../../src/cli/index.js' import type { Options } from '@wdio/types' @@ -29,13 +28,6 @@ vi.mock('../../../src/cli/index.js', () => ({ } })) -// undefined = this worker is not jasmine on the CLI flow, so the existing aggregation applies -vi.mock('../../../src/cli/frameworks/wdioJasmineTestFramework.js', () => ({ - default: { - sessionVerdict: vi.fn(() => undefined) - } -})) - vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ default: { getTrackedInstance: vi.fn(), @@ -1024,6 +1016,11 @@ describe('AutomateModule preferScenarioName', () => { describe('AutomateModule — jasmine session verdict', () => { let automateModule: AutomateModule const putBodies = () => vi.mocked(fetch).mock.calls.map(([url, opts]) => [String(url).split('/sessions/')[1], JSON.parse((opts as { body: string }).body)]) + const inputs = (overrides: Record = {}) => ({ + result: 0, specsRan: true, failReasons: [], pureTestFailReasons: [], hookFailReasons: [], ...overrides + }) + const verdict = (overrides: Record, ignoreHooksStatus: boolean) => + (automateModule as any).jasmineVerdict(inputs(overrides), ignoreHooksStatus) beforeEach(() => { vi.clearAllMocks() @@ -1040,17 +1037,61 @@ describe('AutomateModule — jasmine session verdict', () => { } as any }) - afterEach(() => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReset().mockReturnValue(undefined) + describe('jasmineVerdict (legacy service.after())', () => { + it('passes a clean run with no reason, whatever ignoreHooksStatus says', () => { + expect(verdict({}, false)).toEqual({ status: 'passed' }) + expect(verdict({}, true)).toEqual({ status: 'passed' }) + }) + + it('fails a runner-clean run on any recorded reason, joined in order', () => { + expect(verdict({ failReasons: ['beforeAll hook failure', 'boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['beforeAll hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeAll hook failure\nboom' }) + }) + + it('under ignoreHooksStatus judges a runner-clean run on its pure test failures only', () => { + expect(verdict({ hookFailReasons: ['afterAll hook failure'] }, true)).toEqual({ status: 'passed' }) + expect(verdict({ failReasons: ['boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['afterAll hook failure'] }, true)) + .toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('under ignoreHooksStatus passes a runner-failed run whose only failures are hooks', () => { + expect(verdict({ result: 2, hookFailReasons: ['beforeEach hook failure'] }, true)).toEqual({ status: 'passed' }) + }) + + it('under ignoreHooksStatus fails a runner-failed run with pure failures, on those reasons', () => { + expect(verdict({ result: 1, failReasons: ['boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['hook'] }, true)) + .toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('under ignoreHooksStatus fails a runner-failed run with no recorded failure, with no reason', () => { + expect(verdict({ result: 1 }, true)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('fails a runner-failed run with every recorded reason when hooks count', () => { + expect(verdict({ result: 1, failReasons: ['beforeEach hook failure', 'boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['beforeEach hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeEach hook failure\nboom' }) + expect(verdict({ result: 1 }, false)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('fails a worker where no spec ran, preferring pure reasons under ignoreHooksStatus', () => { + expect(verdict({ specsRan: false }, false)).toEqual({ status: 'failed', reason: undefined }) + expect(verdict({ specsRan: false, failReasons: ['beforeAll hook failure'], hookFailReasons: ['beforeAll hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeAll hook failure' }) + expect(verdict({ specsRan: false, hookFailReasons: ['beforeAll hook failure'] }, true)).toEqual({ status: 'failed', reason: undefined }) + expect(verdict({ specsRan: false, failReasons: ['boom'], pureTestFailReasons: ['boom'] }, true)).toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('reproduces the CP0 hookfail verdict: failed, every hook failure in the order it happened', () => { + const hookFailures = ['beforeAll hook failure', 'beforeEach hook failure', 'beforeEach hook failure', 'afterEach hook failure', 'afterEach hook failure'] + expect(verdict({ result: 6, failReasons: hookFailures, hookFailReasons: hookFailures }, false)) + .toEqual({ status: 'failed', reason: hookFailures.join('\n') }) + }) }) - it('marks the live session with the framework verdict, the last name and the joined reasons', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' - ? { status: 'failed', reason: 'beforeAll hook failure\nafterEach hook failure' } - : null) - ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Hookfail afterEach suite', appliedName: 'Hookfail afterEach suite', testResults: new Map(), scenariosRan: 0 }) + it('marks the live session with the verdict, the last name and the joined reasons', async () => { + (automateModule as any).sessionMap.set('live', { lastTestName: 'Hookfail afterEach suite', appliedName: 'Hookfail afterEach suite', testResults: new Map(), scenariosRan: 0 }) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, failReasons: ['beforeAll hook failure', 'afterEach hook failure'] }) }) expect(putBodies()).toEqual([ ['live.json', { status: 'failed', name: 'Hookfail afterEach suite', reason: 'beforeAll hook failure\nafterEach hook failure' }] @@ -1058,18 +1099,15 @@ describe('AutomateModule — jasmine session verdict', () => { }) it('marks a live session no spec registered, without a name', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'beforeAll hook failure' }) - - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, specsRan: false, failReasons: ['beforeAll hook failure'] }) }) expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'beforeAll hook failure' }]]) }) it('leaves a reloaded session to its onReload mark, but still names it', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' ? { status: 'passed' } : null) - ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + (automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs() }) expect(putBodies()).toEqual([ ['old.json', { name: 'Suite A' }], @@ -1077,32 +1115,39 @@ describe('AutomateModule — jasmine session verdict', () => { ]) }) - it('passes ignoreHooksStatus from the worker\'s service options to the verdict', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'passed' }) + it('marks no session status when no session is live', async () => { + vi.mocked(AutomationFramework.getState).mockReturnValue('' as any) + ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1 }) }) + + expect(putBodies()).toEqual([['old.json', { name: 'Suite A' }]]) + }) + + it('reads ignoreHooksStatus from the worker\'s service options', async () => { vi.mocked(BrowserstackCLI.getInstance).mockReturnValueOnce({ options: { testObservabilityOptions: { ignoreHooksStatus: true } } } as any) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, hookFailReasons: ['beforeEach hook failure'] }) }) - expect(WdioJasmineTestFramework.sessionVerdict).toHaveBeenCalledWith('live', true) + expect(putBodies()).toEqual([['live.json', { status: 'passed' }]]) }) it('honours skipSessionStatus and skipSessionName', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'x' }) - ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', testResults: new Map(), scenariosRan: 0 }) + (automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', testResults: new Map(), scenariosRan: 0 }) ;(automateModule.config as any).testContextOptions = { skipSessionName: true, skipSessionStatus: true } - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, failReasons: ['x'] }) }) expect(fetch).not.toHaveBeenCalled() }) - it('keeps the existing per-test aggregation when the worker is not jasmine', async () => { + it('keeps the existing per-test aggregation when no verdict inputs arrive (mocha, cucumber)', async () => { (automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', appliedName: 'Suite', scenariosRan: 0, testResults: new Map([['t', { testName: 'Suite', status: 'failed', reason: 'boom' }]]) }) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({}) expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'boom' }]]) }) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index dd1e3b9b..25978192 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -8,7 +8,6 @@ import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' import { HookState } from '../../src/cli/states/hookState.js' import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' import TestHubModule from '../../src/cli/modules/testHubModule.js' -import AutomationFramework from '../../src/cli/frameworks/automationFramework.js' vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) @@ -60,6 +59,23 @@ describe('WdioJasmineTestFramework', () => { let testHub: Record> const drain = () => framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, {}) + // The reporter does not await trackEvent: its events are applied before the call returns. + const reporterTestStart = (testStats: unknown, ctx: unknown) => { + const args: Record = { source: 'reporter', testStats, context: ctx } + framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + return args.testUuid as string | undefined + } + const reporterTestEnd = (testStats: unknown, ctx: unknown) => { + framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { source: 'reporter', testStats, context: ctx }) + } + const reporterHook = (hookState: State, hookStats: { title: string }, ctx: unknown) => { + framework.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), hookState, { source: 'reporter', hookStats, context: ctx }) + } + const reporterHookStart = (hookStats: { title: string }, ctx: unknown) => reporterHook(HookState.PRE, hookStats, ctx) + const reporterHookEnd = (hookStats: { title: string }, ctx: unknown = context()) => reporterHook(HookState.POST, hookStats, ctx) + const reporterLog = (logEntry: Record) => { + framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { source: 'reporter', logEntry }) + } beforeEach(() => { process.env.BROWSERSTACK_OBSERVABILITY = 'true' @@ -92,7 +108,7 @@ describe('WdioJasmineTestFramework', () => { }) it('sends legacy identity for a spec, char-for-char with the CP0 TestRun', async () => { - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await drain() expect(dispatches).toHaveLength(1) @@ -116,15 +132,15 @@ describe('WdioJasmineTestFramework', () => { }) it('keeps @tags from describes and the spec, sigil included', async () => { - framework.onReporterTestStart(testStats({ title: 'tagged @smoke', fullTitle: 'Outer @regression tagged @smoke' }) as any, context(['Outer @regression'])) + reporterTestStart(testStats({ title: 'tagged @smoke', fullTitle: 'Outer @regression tagged @smoke' }), context(['Outer @regression'])) await drain() expect(dispatches[0].data.test_tags).toEqual(['@regression', '@smoke']) }) it('finishes a passed spec on the same uuid with reporter timing and the result timestamp', async () => { const stats = testStats() - framework.onReporterTestStart(stats as any, context()) - framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date('2026-09-25T15:34:16.645Z'), _duration: 4386 } as any, context()) + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date('2026-09-25T15:34:16.645Z'), _duration: 4386 }, context()) await drain() expect(dispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) @@ -140,9 +156,9 @@ describe('WdioJasmineTestFramework', () => { }) it('reports a pending spec as Start + Finish(skipped)', async () => { - framework.onReporterTestStart(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test' }) as any, context(['Pending suite'])) + reporterTestStart(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test' }), context(['Pending suite'])) // @wdio/reporter replaces the TestStats object on test:pending; only the uid carries over - framework.onReporterTestEnd(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test', state: 'skipped', end: new Date() }) as any, context(['Pending suite'])) + reporterTestEnd(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test', state: 'skipped', end: new Date() }), context(['Pending suite'])) await drain() expect(dispatches).toHaveLength(2) @@ -153,8 +169,8 @@ describe('WdioJasmineTestFramework', () => { it('keeps jasmine\'s exact reason on a beforeAll-failed child', async () => { const error = { message: BEFORE_ALL_REASON, stack: '' } - framework.onReporterTestStart(testStats({ uid: 'child0' }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'child0', state: 'failed', error, end: new Date() }) as any, context()) + reporterTestStart(testStats({ uid: 'child0' }), context()) + reporterTestEnd(testStats({ uid: 'child0', state: 'failed', error, end: new Date() }), context()) await drain() const finish = dispatches[1].data @@ -165,17 +181,17 @@ describe('WdioJasmineTestFramework', () => { }) it('maps an AssertionError message to failure_type AssertionError', async () => { - framework.onReporterTestStart(testStats() as any, context()) - framework.onReporterTestEnd(testStats({ state: 'failed', error: { message: 'AssertionError: nope', stack: 'at x' } }) as any, context()) + reporterTestStart(testStats(), context()) + reporterTestEnd(testStats({ state: 'failed', error: { message: 'AssertionError: nope', stack: 'at x' } }), context()) await drain() expect(dispatches[1].data.test_failure_type).toBe('AssertionError') }) it('gives overlapping specs distinct instances and closes each on its own uuid', async () => { - framework.onReporterTestStart(testStats({ uid: 'a' }) as any, context()) - framework.onReporterTestStart(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b' }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'a', state: 'passed', end: new Date() }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b', state: 'passed', end: new Date() }) as any, context()) + reporterTestStart(testStats({ uid: 'a' }), context()) + reporterTestStart(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b' }), context()) + reporterTestEnd(testStats({ uid: 'a', state: 'passed', end: new Date() }), context()) + reporterTestEnd(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b', state: 'passed', end: new Date() }), context()) await drain() const uuidOf = (i: number) => dispatches[i].data.test_uuid @@ -187,11 +203,11 @@ describe('WdioJasmineTestFramework', () => { it('reports beforeAll/afterAll with legacy hook identity and no test linkage', async () => { const before = hookStats('"before all" hook') - framework.onReporterHookStart(before as any, context()) - framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date('2026-09-25T15:34:12.257Z'), _duration: 4087 } as any) + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'passed', end: new Date('2026-09-25T15:34:12.257Z'), _duration: 4087 }) const after = hookStats('"after all" hook') - framework.onReporterHookStart(after as any, context(['Nested outer', 'Nested middle'])) - framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) + reporterHookStart(after, context(['Nested outer', 'Nested middle'])) + reporterHookEnd({ ...after, state: 'passed', end: new Date() }) await drain() expect(dispatches.map(d => [d.state, d.hook])).toEqual([ @@ -228,8 +244,8 @@ describe('WdioJasmineTestFramework', () => { it('carries failure fields on a failed beforeAll', async () => { const before = hookStats('"before all" hook') - framework.onReporterHookStart(before as any, context()) - framework.onReporterHookEnd({ ...before, state: 'failed', error: { message: 'boom', stack: 'Error: boom' }, end: new Date() } as any) + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'failed', error: { message: 'boom', stack: 'Error: boom' }, end: new Date() }) await drain() const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] expect(finished).toMatchObject({ @@ -241,8 +257,8 @@ describe('WdioJasmineTestFramework', () => { }) it('never reports each-hooks, or a hook outside any describe', async () => { - framework.onReporterHookStart(hookStats('"before each" hook') as any, context()) - framework.onReporterHookStart(hookStats('"before all" hook') as any, context([])) + reporterHookStart(hookStats('"before each" hook'), context()) + reporterHookStart(hookStats('"before all" hook'), context([])) await drain() expect(dispatches).toHaveLength(0) }) @@ -252,10 +268,10 @@ describe('WdioJasmineTestFramework', () => { process.env.BROWSERSTACK_ACCESSIBILITY = 'true' try { const before = hookStats('"before all" hook') - framework.onReporterHookStart(before as any, context()) - framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date() } as any) - framework.onReporterTestStart(testStats() as any, context()) - framework.onReporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'passed', end: new Date() }) + reporterTestStart(testStats(), context()) + reporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) await drain() expect(dispatches.map(d => d.state)).toEqual([TestFrameworkState.TEST]) expect(logSends).toHaveLength(0) @@ -266,8 +282,8 @@ describe('WdioJasmineTestFramework', () => { it('sends reporter events to TestHub only, never through the module observers', async () => { const stats = testStats() - framework.onReporterTestStart(stats as any, context()) - framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date() }, context()) await drain() expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(2) expect(sessionEvents).toBe(1) @@ -276,7 +292,7 @@ describe('WdioJasmineTestFramework', () => { it('drives the modules from the service hooks on the reporter\'s instance, with TestHub skipped', async () => { const spec = { description: 'outer passing test', fullName: 'Nested outer outer passing test' } - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) const uuid = dispatches[0].data.test_uuid expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) @@ -297,8 +313,8 @@ describe('WdioJasmineTestFramework', () => { }) it('never shows a spec the reporter saw but the service did not (pending, excluded, beforeAll-failed) to the modules', async () => { - framework.onReporterTestStart(testStats({ uid: 'x' }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'x', state: 'skipped', end: new Date() }) as any, context()) + reporterTestStart(testStats({ uid: 'x' }), context()) + reporterTestEnd(testStats({ uid: 'x', state: 'skipped', end: new Date() }), context()) await drain() expect(dispatches).toHaveLength(2) expect(moduleDispatches).toHaveLength(0) @@ -316,13 +332,13 @@ describe('WdioJasmineTestFramework', () => { }) it('sends an open all-hook\'s logs in the hook state, else the last-started spec\'s in the test state', async () => { - framework.onReporterTestStart(testStats() as any, context()) - framework.onReporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) + reporterTestStart(testStats(), context()) + reporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) const after = hookStats('"after all" hook') - framework.onReporterHookStart(after as any, context()) - framework.onReporterLog({ level: 'INFO', message: 'in hook', timestamp: 't2', kind: 'TEST_LOG' }) - framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) - framework.onReporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) + reporterHookStart(after, context()) + reporterLog({ level: 'INFO', message: 'in hook', timestamp: 't2', kind: 'TEST_LOG' }) + reporterHookEnd({ ...after, state: 'passed', end: new Date() }) + reporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) await drain() expect(logSends.map(l => l.state)).toEqual(['TestFrameworkState.TEST', 'TestFrameworkState.AFTER_ALL', 'TestFrameworkState.TEST']) @@ -335,7 +351,7 @@ describe('WdioJasmineTestFramework', () => { }) it('keeps a screenshot entry\'s kind on the log path', async () => { - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) await drain() expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') @@ -343,10 +359,10 @@ describe('WdioJasmineTestFramework', () => { it('sends an HTTP command log to the spec it names, even inside an all-hook or after the spec ended', async () => { const first = testStats() - const uuid = framework.onReporterTestStart(first as any, context()) - framework.onReporterTestEnd({ ...first, state: 'passed', end: new Date() } as any, context()) + const uuid = reporterTestStart(first, context()) + reporterTestEnd({ ...first, state: 'passed', end: new Date() }, context()) const after = hookStats('"after all" hook') - framework.onReporterHookStart(after as any, context()) + reporterHookStart(after, context()) const message = JSON.stringify({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message, timestamp: 't', test_run_uuid: uuid } }) await drain() @@ -360,7 +376,7 @@ describe('WdioJasmineTestFramework', () => { }) it('drops a named-spec log whose uuid this worker never minted', async () => { - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message: '{}', timestamp: 't', test_run_uuid: 'not-ours' } }) await drain() expect(logSends).toHaveLength(0) @@ -381,102 +397,54 @@ describe('WdioJasmineTestFramework', () => { }) it('is not idle until queued events are dispatched', async () => { - framework.onReporterTestStart(testStats() as any, context()) - expect(WdioJasmineTestFramework.isIdle()).toBe(false) + reporterTestStart(testStats(), context()) + expect(framework.isIdle()).toBe(false) await drain() - expect(WdioJasmineTestFramework.isIdle()).toBe(true) + expect(framework.isIdle()).toBe(true) }) - it('logs and continues when an observer throws', async () => { - testHub.sendTestFrameworkEvent.mockRejectedValueOnce(new Error('send blew up')) - const stats = testStats() - framework.onReporterTestStart(stats as any, context()) - framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + it('counts pending events per framework instance', async () => { + const other = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') + reporterTestStart(testStats(), context()) + expect(framework.isIdle()).toBe(false) + expect(other.isIdle()).toBe(true) await drain() - expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) - expect(WdioJasmineTestFramework.isIdle()).toBe(true) }) - describe('session verdict (legacy service.after())', () => { - let liveSession = 'live' - const spec = (title: string, overrides: Record = {}) => testStats({ uid: title, title, fullTitle: `Suite ${title}`, ...overrides }) - const reporterEnd = (stats: ReturnType, state: string) => - framework.onReporterTestEnd({ ...stats, state, end: new Date() } as any, context()) - const serviceTest = (title: string, result: Record) => - framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: { fullName: `Suite ${title}`, description: title }, result }) - const serviceHook = (state: State, result: Record) => framework.trackEvent(state, HookState.POST, { test: {}, result }) - const fail = (message: string) => ({ passed: false, error: new Error(message) }) - - beforeEach(() => { - liveSession = 'live' - vi.spyOn(AutomationFramework, 'getTrackedInstance').mockReturnValue({} as any) - vi.spyOn(AutomationFramework, 'getState').mockImplementation(() => liveSession) - }) - - it('reproduces the CP0 hookfail verdict: failed, every hook failure in the order it happened', async () => { - await serviceHook(TestFrameworkState.BEFORE_ALL, fail('beforeAll hook failure')) - for (const title of ['beforeAll child one', 'beforeAll child two', 'beforeEach child one', 'beforeEach child two']) { - if (title.startsWith('beforeEach')) { - await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) - } - const stats = spec(title) - framework.onReporterTestStart(stats as any, context()) - reporterEnd(stats, 'failed') - } - for (const title of ['afterEach child one', 'afterEach child two']) { - const stats = spec(title) - framework.onReporterTestStart(stats as any, context()) - await serviceTest(title, { passed: true }) - await serviceHook(TestFrameworkState.AFTER_EACH, fail('afterEach hook failure')) - reporterEnd(stats, 'failed') - } - await drain() - - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ - status: 'failed', - reason: 'beforeAll hook failure\nbeforeEach hook failure\nbeforeEach hook failure\nafterEach hook failure\nafterEach hook failure' - }) - }) - - it('fails a session where no spec ran, even with no failure recorded', async () => { - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: undefined }) - }) - - it('passes when specs ran and nothing failed, with no reason', async () => { - const stats = spec('ok') - framework.onReporterTestStart(stats as any, context()) - await serviceTest('ok', { passed: true }) - reporterEnd(stats, 'passed') - await drain() - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) - }) + it('mints a reporter spec before trackEvent returns, so the uuid and the service lookup are immediate', async () => { + const args: Record = { source: 'reporter', testStats: testStats(), context: context() } + const pending = framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + const uuid = args.testUuid + expect(uuid).toEqual(expect.any(String)) + expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) + expect(dispatches).toHaveLength(0) - it('does not count a pending() spec as a failure', async () => { - await serviceTest('pending', { passed: false, skipped: true }) - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) - }) + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: { description: 'outer passing test', fullName: 'Nested outer outer passing test' } }) + await pending + expect(process.env.TEST_ANALYTICS_ID).toBe(uuid) + expect(dispatches[0].data.test_uuid).toBe(uuid) + }) - it('under ignoreHooksStatus passes a run whose only failures are hooks, and keeps test failures', async () => { - await serviceTest('ok', { passed: true }) - await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) - const child = spec('child') - framework.onReporterTestStart(child as any, context()) - reporterEnd(child, 'failed') - await drain() - expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'passed' }) - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: 'beforeEach hook failure' }) + it('leaves no uuid on the args when minting fails, and logs instead of throwing', async () => { + const args: Record = { source: 'reporter', testStats: undefined, context: context() } + await expect(framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args)).resolves.toBeUndefined() + expect(args.testUuid).toBeUndefined() + expect(framework.isIdle()).toBe(true) + }) - await serviceTest('bad', fail('boom')) - expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'failed', reason: 'boom' }) - }) + it('reports a hook in BEFORE_ALL/AFTER_ALL and every other hook in NONE', () => { + expect(['"before all" hook', '"after all" hook', '"before each" hook', '"after each" hook', undefined].map(t => WdioJasmineTestFramework.reporterHookState(t))) + .toEqual([TestFrameworkState.BEFORE_ALL, TestFrameworkState.AFTER_ALL, TestFrameworkState.NONE, TestFrameworkState.NONE, TestFrameworkState.NONE]) + }) - it('keeps failures per session and gives no verdict for a session that is no longer live', async () => { - await serviceTest('first', fail('before reload')) - liveSession = 'reloaded' - await serviceTest('second', { passed: true }) - expect(WdioJasmineTestFramework.sessionVerdict('reloaded', false)).toEqual({ status: 'passed' }) - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toBeNull() - }) + it('logs and continues when an observer throws', async () => { + testHub.sendTestFrameworkEvent.mockRejectedValueOnce(new Error('send blew up')) + const stats = testStats() + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date() }, context()) + await drain() + expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) + expect(framework.isIdle()).toBe(true) }) // The class calls these TestHubModule methods directly; renaming or removing one must fail here. diff --git a/packages/browserstack-service/tests/reporter.test.ts b/packages/browserstack-service/tests/reporter.test.ts index 2714f675..5f8a169a 100644 --- a/packages/browserstack-service/tests/reporter.test.ts +++ b/packages/browserstack-service/tests/reporter.test.ts @@ -6,6 +6,8 @@ import type { StdLog } from '../src/index.js' import TestReporter from '../src/reporter.js' import { BrowserstackCLI } from '../src/cli/index.js' import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' @@ -385,7 +387,8 @@ describe('test-reporter', () => { let framework: WdioJasmineTestFramework let getInstanceSpy: ReturnType | undefined let getGitMetaDataSpy: ReturnType - let isIdleSpy: ReturnType | undefined + // the reporter-sourced trackEvent calls, as [state, hookState, args] + const reporterCalls = () => vi.mocked(framework.trackEvent).mock.calls.filter(([, , args]) => (args as Record)?.source === 'reporter') const setCli = (running: boolean, testFramework: unknown) => { getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ @@ -398,14 +401,11 @@ describe('test-reporter', () => { vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') getGitMetaDataSpy = vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') - for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { - vi.spyOn(framework, m) - } - vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') - vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterLog).mockReturnValue(undefined) + vi.spyOn(framework, 'trackEvent').mockImplementation(async (state, hookState, args = {}) => { + if (state === TestFrameworkState.TEST && hookState === HookState.PRE) { + args.testUuid = 'cli-uuid' + } + }) reporter = new TestReporter({}) await reporter.onRunnerStart(jasmineRunnerConfig as any) @@ -422,8 +422,6 @@ describe('test-reporter', () => { afterEach(() => { getInstanceSpy?.mockRestore() getInstanceSpy = undefined - isIdleSpy?.mockRestore() - isIdleSpy = undefined getGitMetaDataSpy.mockRestore() for (const spy of Object.values(listener)) { spy.mockRestore() @@ -441,11 +439,14 @@ describe('test-reporter', () => { await reporter.onTestEnd({ ...stats, state: 'passed' } as any) await reporter.appendTestItemLog(logEntry()) - expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterLog).toHaveBeenCalledTimes(1) + expect(reporterCalls().map(([state, hookState]) => [state, hookState])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + [TestFrameworkState.TEST, HookState.PRE], + [TestFrameworkState.TEST, HookState.POST], + [TestFrameworkState.LOG, HookState.POST], + ]) + expect(framework.trackEvent).toHaveBeenCalledTimes(5) for (const spy of Object.values(listener)) { expect(spy).not.toHaveBeenCalled() } @@ -453,10 +454,38 @@ describe('test-reporter', () => { it('passes the suite stack and suite file', async () => { await reporter.onTestStart(jasmineTestStats() as any) - expect(framework.onReporterTestStart).toHaveBeenCalledWith( - expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), - { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } - ) + expect(framework.trackEvent).toHaveBeenCalledWith(TestFrameworkState.TEST, HookState.PRE, expect.objectContaining({ + source: 'reporter', + testStats: expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), + context: { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' }, + })) + }) + + it('passes hook stats with the suite context, and log entries as they came', async () => { + const hook = hookStats() + await reporter.onHookStart(hook as any) + await reporter.onHookEnd(hook as any) + const entry = logEntry() + await reporter.appendTestItemLog(entry) + const context = { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } + expect(reporterCalls().map(([, , args]) => args)).toEqual([ + { source: 'reporter', hookStats: hook, context }, + { source: 'reporter', hookStats: hook, context }, + { source: 'reporter', logEntry: entry }, + ]) + }) + + it('reports an each-hook in NONE, leaving the class to drop it', async () => { + await reporter.onHookStart({ ...hookStats(), title: '"before each" hook' } as any) + expect(framework.trackEvent).toHaveBeenCalledWith(TestFrameworkState.NONE, HookState.PRE, expect.objectContaining({ source: 'reporter' })) + }) + + it('records nothing for the spec when the framework minted no uuid', async () => { + vi.mocked(framework.trackEvent).mockResolvedValue(undefined) + ;(TestReporter as any).currentTest = {} + await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: 'No uuid spec' } as any) + expect(TestReporter.getTests()['No uuid spec']).toBeUndefined() + expect((TestReporter as any).currentTest).toEqual({}) }) it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { @@ -481,14 +510,13 @@ describe('test-reporter', () => { it('drops ', async () => { await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: '' } as any) await reporter.onTestEnd({ ...jasmineTestStats(), fullTitle: '' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterTestEnd).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() }) it('sends nothing when Test Observability is opted out', async () => { reporter['_observability'] = false await reporter.onTestStart(jasmineTestStats() as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() expect(listener.testStarted).not.toHaveBeenCalled() }) @@ -501,9 +529,14 @@ describe('test-reporter', () => { }) it('reports unsynchronised while the framework has queued events', () => { - isIdleSpy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) + const isIdle = vi.spyOn(framework, 'isIdle').mockReturnValue(false) expect(reporter.isSynchronised).toBe(false) - isIdleSpy.mockReturnValue(true) + isIdle.mockReturnValue(true) + expect(reporter.isSynchronised).toBe(true) + }) + + it('is synchronised when the CLI test framework is not jasmine\'s', () => { + setCli(true, { isIdle: () => false }) expect(reporter.isSynchronised).toBe(true) }) }) @@ -525,8 +558,7 @@ describe('test-reporter', () => { expect(listener.testStarted).toHaveBeenCalledTimes(1) expect(listener.testFinished).toHaveBeenCalledTimes(1) expect(listener.logCreated).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterLog).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: '123456789' }) }) }) @@ -538,7 +570,7 @@ describe('test-reporter', () => { reporter['_config']!.framework = 'mocha' await reporter.onTestStart(jasmineTestStats() as any) await reporter.onTestEnd({ ...jasmineTestStats(), state: 'passed' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() expect(listener.testStarted).not.toHaveBeenCalled() expect(listener.testFinished).not.toHaveBeenCalled() }) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 15907aa6..a9b697bd 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -17,6 +17,7 @@ import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' import { HookState } from '../src/cli/states/hookState.js' import { AutomationFrameworkConstants } from '../src/cli/frameworks/constants/automationFrameworkConstants.js' import { AutomationFrameworkState } from '../src/cli/states/automationFrameworkState.js' +import PerformanceTester from '../src/instrumentation/performance/performance-tester.js' const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' const sessionBaseUrl = 'https://api.browserstack.com/automate/sessions' @@ -3275,3 +3276,65 @@ describe('_cucumberTestResult failure reason adjacent', () => { expect(result.error?.message).toBe('Unknown Error') }) }) + +describe('after hands AutomateModule the jasmine session verdict inputs', () => { + // describe('after') swaps the prototype method for a stand-in; these cases need the real one + const after = BrowserstackService.prototype.after + let trackEvent: ReturnType + let getInstanceSpy: ReturnType | undefined + + const makeService = (framework: string) => new BrowserstackService( + { testObservability: false, setSessionStatus: true } as never, + [] as never, + { user: 'foo', key: 'bar', framework } as never + ) + const executeArgs = () => trackEvent.mock.calls.find(([state, hook]) => state === AutomationFrameworkState.EXECUTE && hook === HookState.POST)?.[2] + + beforeEach(() => { + trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent }), + modules: {} + } as never) + ;(PerformanceTester as unknown as Record).stopAndGenerate = vi.fn().mockResolvedValue(undefined) + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + delete (PerformanceTester as unknown as Record).stopAndGenerate + }) + + it.each(['mocha', 'cucumber'])('sends nothing extra for %s', async (framework) => { + const service = makeService(framework) + service['_specsRan'] = true + service['_failReasons'] = ['boom'] + + await after.call(service, 1) + + expect(executeArgs()).toEqual({}) + }) + + it('sends the runner result and a copy of the tracked reasons for jasmine', async () => { + const service = makeService('jasmine') + service['_specsRan'] = true + service['_failReasons'] = ['beforeEach hook failure', 'boom'] + service['_pureTestFailReasons'] = ['boom'] + service['_hookFailReasons'] = ['beforeEach hook failure'] + + await after.call(service, 2) + + const args = executeArgs() as { sessionVerdictInputs: Record } + expect(args).toEqual({ + sessionVerdictInputs: { + result: 2, + specsRan: true, + failReasons: ['beforeEach hook failure', 'boom'], + pureTestFailReasons: ['boom'], + hookFailReasons: ['beforeEach hook failure'], + } + }) + expect(args.sessionVerdictInputs.failReasons).not.toBe(service['_failReasons']) + }) +}) From 7d13ba0202af5f835ff0490baf7aa4b82ec93c52 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 7 Oct 2026 16:05:00 +0530 Subject: [PATCH 15/15] refactor(cli): drop the jasmine framework's #iso helper for optional-chained toISOString (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../src/cli/frameworks/wdioJasmineTestFramework.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 5b983ebe..ff4041ab 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -181,7 +181,7 @@ export default class WdioJasmineTestFramework extends TestFramework { [TestFrameworkConstants.KEY_TEST_SCOPES]: scopes, [TestFrameworkConstants.KEY_TEST_TAGS]: getTestTags(testStats as unknown as Frameworks.Test, scopes), ...this.#filePaths(context.suiteFile), - [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: testStats.start?.toISOString(), [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, [WdioJasmineTestFramework.KEY_TEST_RETRIES]: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, }) @@ -226,8 +226,8 @@ export default class WdioJasmineTestFramework extends TestFramework { } const entries: Record = { [TestFrameworkConstants.KEY_TEST_RESULT]: result, - [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), - [TestFrameworkConstants.KEY_TEST_ENDED_AT]: this.#iso(testStats.end), + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: testStats.start?.toISOString(), + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: testStats.end?.toISOString(), [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), } @@ -268,7 +268,7 @@ export default class WdioJasmineTestFramework extends TestFramework { key, [TestFrameworkConstants.KEY_HOOK_ID]: hookId, [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, - [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: this.#iso(hookStats.start), + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: hookStats.start?.toISOString(), [TestFrameworkConstants.KEY_HOOK_LOGS]: [], [TestFrameworkConstants.KEY_HOOK_NAME]: title, [WdioJasmineTestFramework.KEY_HOOK_IDENTIFIER]: `${title} for ${scopes[scopes.length - 1]}`, @@ -316,7 +316,7 @@ export default class WdioJasmineTestFramework extends TestFramework { failure = this.#failureFields(error) } } - const endedAt = this.#iso(hookStats.end) + const endedAt = hookStats.end?.toISOString() const duration = hookStats._duration const emitHook = shouldProcessEventForTesthub('HookRunFinished') this.#enqueue(`${key}/POST`, async () => { @@ -484,10 +484,6 @@ export default class WdioJasmineTestFramework extends TestFramework { return fullName.slice(0, fullName.indexOf(testStats.title || '') - 1) } - #iso(date: Date | undefined) { - return date ? date.toISOString() : undefined - } - static #hookState(key: string) { if (key === 'BEFORE_ALL') { return TestFrameworkState.BEFORE_ALL