Skip to content

Commit cd37699

Browse files
riglarriglar
authored
Merge pull request #5 from moropo-com/test/tighten-integration-assertions
test: make integration suites assert deterministic outcomes
2 parents 2bf88a1 + da74f89 commit cd37699

7 files changed

Lines changed: 684 additions & 1539 deletions

‎src/methods.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { ux } from './utils/progress';
22
import { createHash } from 'node:crypto';
3-
import { createReadStream, readdirSync, writeFileSync } from 'node:fs';
3+
import { createReadStream, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
44
import { access, readFile } from 'node:fs/promises';
55
import * as path from 'node:path';
66
import * as StreamZip from 'node-stream-zip';
@@ -1064,6 +1064,10 @@ export const writeJSONFile = (
10641064
logger: { log: (message: string) => void; warn: (message: string) => void },
10651065
) => {
10661066
try {
1067+
const directory = path.dirname(filePath);
1068+
if (directory !== '.') {
1069+
mkdirSync(directory, { recursive: true });
1070+
}
10671071
writeFileSync(filePath, JSON.stringify(data, null, 2));
10681072
logger.log(colors.dim('JSON output written to: ') + colors.highlight(path.resolve(filePath)));
10691073
} catch (error) {

‎test/integration/artifacts.integration.test.ts‎

Lines changed: 107 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -1,147 +1,156 @@
11
import { expect } from 'chai';
2-
import { exec as execCallback } from 'node:child_process';
3-
import { promisify } from 'node:util';
2+
import * as fs from 'node:fs';
3+
import * as os from 'node:os';
4+
import * as path from 'node:path';
5+
6+
import {
7+
CLI,
8+
DEAD_API_URL,
9+
MOCK_API_KEY,
10+
MOCK_API_URL,
11+
exec,
12+
runExpectingFailure,
13+
} from './helpers';
414

5-
const exec = promisify(execCallback);
15+
describe('Artifacts Command Integration Tests', () => {
16+
const mockApiUrl = MOCK_API_URL;
17+
const mockApiKey = MOCK_API_KEY;
18+
const mockUploadId = '123e4567-e89b-12d3-a456-426614174000';
19+
// Downloads write into cwd, so keep them out of the repo tree.
20+
let tempDir: string;
621

7-
const run = (args: string, env?: Record<string, string>) =>
8-
exec(`./dist/index.js artifacts ${args}`, {
9-
env: { ...process.env, ...env },
10-
timeout: 15_000,
22+
before(() => {
23+
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dcd-artifacts-test-'));
1124
});
1225

13-
const errorOutput = (error: unknown): string => {
14-
if (error && typeof error === 'object') {
15-
if ('stderr' in error && typeof error.stderr === 'string' && error.stderr) return error.stderr;
16-
if ('stdout' in error && typeof error.stdout === 'string' && error.stdout) return error.stdout;
17-
}
26+
after(() => {
27+
if (fs.existsSync(tempDir)) {
28+
fs.rmSync(tempDir, { force: true, recursive: true });
29+
}
30+
});
1831

19-
return '';
20-
};
32+
const run = (args: string, env?: Record<string, string>) =>
33+
exec(`${CLI} artifacts ${args}`, {
34+
cwd: tempDir,
35+
env: { ...process.env, ...env },
36+
timeout: 15_000,
37+
});
2138

22-
describe('Artifacts Command Integration Tests', () => {
23-
const mockApiUrl = 'http://localhost:3001';
24-
const mockApiKey = 'test-api-key-123';
25-
const mockUploadId = '123e4567-e89b-12d3-a456-426614174000';
39+
const runFailing = (args: string, env?: Record<string, string>) =>
40+
runExpectingFailure(`${CLI} artifacts ${args}`, {
41+
cwd: tempDir,
42+
env: { ...process.env, ...env },
43+
});
2644

2745
describe('flag validation', () => {
2846
it('should require --upload-id', async () => {
29-
try {
30-
await run(`--download-artifacts FAILED --api-key ${mockApiKey} --api-url ${mockApiUrl}`);
31-
expect.fail('Should have thrown');
32-
} catch (error) {
33-
expect(errorOutput(error)).to.match(/upload-id/i);
34-
}
47+
const { output } = await runFailing(
48+
`--download-artifacts FAILED --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
49+
);
50+
expect(output).to.match(/upload-id/i);
3551
});
3652

3753
it('should require either --download-artifacts or --report', async () => {
38-
try {
39-
await run(`--upload-id ${mockUploadId} --api-key ${mockApiKey} --api-url ${mockApiUrl}`);
40-
expect.fail('Should have thrown');
41-
} catch (error) {
42-
expect(errorOutput(error)).to.match(/download-artifacts|report/i);
43-
}
54+
const { output } = await runFailing(
55+
`--upload-id ${mockUploadId} --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
56+
);
57+
expect(output).to.match(/download-artifacts|report/i);
4458
});
4559

4660
it('should reject --download-artifacts and --report together', async () => {
47-
try {
48-
await run(
49-
`--upload-id ${mockUploadId} --download-artifacts FAILED --report junit --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
50-
);
51-
expect.fail('Should have thrown');
52-
} catch (error) {
53-
expect(errorOutput(error)).to.match(/cannot also be provided/i);
54-
}
61+
const { output } = await runFailing(
62+
`--upload-id ${mockUploadId} --download-artifacts FAILED --report junit --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
63+
);
64+
expect(output).to.match(/cannot also be provided/i);
5565
});
5666

5767
it('should reject --artifacts-path without --download-artifacts', async () => {
58-
try {
59-
await run(
60-
`--upload-id ${mockUploadId} --artifacts-path ./out.zip --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
61-
);
62-
expect.fail('Should have thrown');
63-
} catch (error) {
64-
expect(errorOutput(error)).to.match(/artifacts-path|download-artifacts/i);
65-
}
68+
const { output } = await runFailing(
69+
`--upload-id ${mockUploadId} --artifacts-path ./out.zip --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
70+
);
71+
expect(output).to.match(/artifacts-path|download-artifacts/i);
6672
});
6773

6874
it('should reject --junit-path without --report', async () => {
69-
try {
70-
await run(
71-
`--upload-id ${mockUploadId} --junit-path ./report.xml --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
72-
);
73-
expect.fail('Should have thrown');
74-
} catch (error) {
75-
expect(errorOutput(error)).to.match(/junit-path|report/i);
76-
}
75+
const { output } = await runFailing(
76+
`--upload-id ${mockUploadId} --junit-path ./report.xml --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
77+
);
78+
expect(output).to.match(/junit-path|report/i);
7779
});
7880

7981
it('should only accept ALL or FAILED for --download-artifacts', async () => {
80-
try {
81-
await run(
82-
`--upload-id ${mockUploadId} --download-artifacts SOME --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
83-
);
84-
expect.fail('Should have thrown');
85-
} catch (error) {
86-
expect(errorOutput(error)).to.match(/all|failed|expected.*to be one of/i);
87-
}
82+
const { output } = await runFailing(
83+
`--upload-id ${mockUploadId} --download-artifacts SOME --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
84+
);
85+
expect(output).to.match(/all|failed|expected.*to be one of/i);
8886
});
8987
});
9088

9189
describe('authentication', () => {
9290
it('should require an API key', async () => {
93-
try {
94-
await run(
95-
`--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
96-
{ DEVICE_CLOUD_API_KEY: '' },
97-
);
98-
expect.fail('Should have thrown');
99-
} catch (error) {
100-
expect(errorOutput(error)).to.match(/api key/i);
101-
}
91+
const { output } = await runFailing(
92+
`--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
93+
{ DEVICE_CLOUD_API_KEY: '' },
94+
);
95+
expect(output).to.match(/api key/i);
10296
});
10397

10498
it('should accept API key from environment variable', async () => {
105-
try {
106-
await run(
107-
`--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
108-
{ DEVICE_CLOUD_API_KEY: mockApiKey },
109-
);
110-
} catch (error) {
111-
// API unreachable is fine — the key was accepted if we don't see the key error
112-
expect(errorOutput(error)).to.not.match(/api key is required/i);
113-
}
99+
// Download failures against the mock are warnings, not errors — the
100+
// command exits 0 once the key is accepted (see download behaviour below).
101+
const { stderr } = await run(
102+
`--upload-id ${mockUploadId} --download-artifacts FAILED --api-url ${mockApiUrl}`,
103+
{ DEVICE_CLOUD_API_KEY: mockApiKey },
104+
);
105+
expect(stderr).to.not.match(/api key is required/i);
106+
});
107+
});
108+
109+
describe('download behaviour against the mock API', () => {
110+
// Prism can't serve the binary download endpoints, so the deterministic
111+
// outcome is a warning and exit 0 — download failures must not fail the
112+
// command or crash.
113+
it('should warn and exit 0 when artifacts download fails', async () => {
114+
const { stderr } = await run(
115+
`--upload-id ${mockUploadId} --download-artifacts FAILED --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
116+
);
117+
expect(stderr).to.include('Failed to download artifacts');
118+
});
119+
120+
it('should download the junit report', async () => {
121+
// Unlike the artifacts zip, Prism can serve the junit report endpoint.
122+
const { stdout } = await run(
123+
`--upload-id ${mockUploadId} --report junit --api-key ${mockApiKey} --api-url ${mockApiUrl}`,
124+
);
125+
expect(stdout).to.include('JUNIT test report has been downloaded');
126+
expect(fs.existsSync(path.join(tempDir, 'report.xml'))).to.be.true;
114127
});
115128
});
116129

117130
describe('network error handling', () => {
118131
it('should handle unreachable API gracefully for --download-artifacts', async () => {
119-
try {
120-
await run(
121-
`--upload-id ${mockUploadId} --download-artifacts FAILED --api-key ${mockApiKey} --api-url http://localhost:9999`,
122-
);
123-
} catch (error) {
124-
// Should warn, not crash with an uncaught exception
125-
const out = errorOutput(error);
126-
expect(out).to.not.match(/typeerror|unhandledpromiserejection/i);
127-
}
132+
// Should warn, not crash with an uncaught exception
133+
const { stderr, stdout } = await run(
134+
`--upload-id ${mockUploadId} --download-artifacts FAILED --api-key ${mockApiKey} --api-url ${DEAD_API_URL}`,
135+
);
136+
expect(stderr + stdout).to.not.match(/typeerror|unhandledpromiserejection/i);
137+
expect(stderr).to.include('Failed to download artifacts');
128138
});
129139

130140
it('should handle unreachable API gracefully for --report', async () => {
131-
try {
132-
await run(
133-
`--upload-id ${mockUploadId} --report junit --api-key ${mockApiKey} --api-url http://localhost:9999`,
134-
);
135-
} catch (error) {
136-
const out = errorOutput(error);
137-
expect(out).to.not.match(/typeerror|unhandledpromiserejection/i);
138-
}
141+
const { stderr, stdout } = await run(
142+
`--upload-id ${mockUploadId} --report junit --api-key ${mockApiKey} --api-url ${DEAD_API_URL}`,
143+
);
144+
expect(stderr + stdout).to.not.match(/typeerror|unhandledpromiserejection/i);
145+
expect(stderr).to.match(/failed to download/i);
139146
});
140147
});
141148

142149
describe('help', () => {
143150
it('should display help with all expected flags', async () => {
144-
const { stdout } = await exec('./dist/index.js artifacts --help', { timeout: 10_000 });
151+
const { stdout } = await exec(`${CLI} artifacts --help`, {
152+
timeout: 10_000,
153+
});
145154
expect(stdout).to.include('--upload-id');
146155
expect(stdout).to.include('--download-artifacts');
147156
expect(stdout).to.include('--artifacts-path');

0 commit comments

Comments
 (0)