diff --git a/internal/shrinkwrap-extractor/lib/convertPackageLockToShrinkwrap.js b/internal/shrinkwrap-extractor/lib/convertPackageLockToShrinkwrap.js index 2493f16c5ca..7da25d19b18 100644 --- a/internal/shrinkwrap-extractor/lib/convertPackageLockToShrinkwrap.js +++ b/internal/shrinkwrap-extractor/lib/convertPackageLockToShrinkwrap.js @@ -1,4 +1,5 @@ -import {readFile} from "node:fs/promises"; +import {readFile, mkdtemp, writeFile, rm} from "node:fs/promises"; +import {tmpdir} from "node:os"; import path from "path"; import {Arborist} from "@npmcli/arborist"; import pacote from "pacote"; @@ -48,105 +49,204 @@ export default async function convertPackageLockToShrinkwrap(workspaceRootDir, t path: workspaceRootDir, }); const tree = await arb.loadVirtual(); - const tops = Array.from(tree.tops.values()); - const cliNode = tops.find((node) => node.packageName === targetPackageName); - if (!cliNode) { + let targetNode = tree.inventory.get(`node_modules/${targetPackageName}`); + if (!targetNode) { throw new Error(`Target package "${targetPackageName}" not found in workspace`); } + targetNode = targetNode.isLink ? targetNode.target : targetNode; - const relevantPackageLocations = new Map(); + const virtualFlatTree = []; // Collect all package keys using arborist - collectDependencies(cliNode, relevantPackageLocations); - - // Using the keys, extract relevant package-entries from package-lock.json - const extractedPackages = Object.create(null); - for (let [packageLoc, node] of relevantPackageLocations) { - let pkg = packageLockJson.packages[packageLoc]; - if (pkg.link) { - pkg = packageLockJson.packages[pkg.resolved]; - } - if (pkg.name === targetPackageName) { - // Make the target package the root package - packageLoc = ""; - if (extractedPackages[packageLoc]) { - throw new Error(`Duplicate root package entry for "${targetPackageName}"`); - } - } else { - packageLoc = normalizePackageLocation(packageLoc, node, targetPackageName, tree.packageName); - } - if (packageLoc !== "" && !pkg.resolved) { - // For all but the root package, ensure that "resolved" and "integrity" fields are present - // These are always missing for locally linked packages, but sometimes also for others (e.g. if installed - // from local cache) - const {resolved, integrity} = await fetchPackageMetadata(node.packageName, node.version, workspaceRootDir); - pkg.resolved = resolved; - pkg.integrity = integrity; - } - extractedPackages[packageLoc] = pkg; - } + resolveVirtualTree(targetNode, virtualFlatTree); + + const physicalTree = new Map(); + await buildPhysicalTree( + virtualFlatTree, physicalTree, packageLockJson, workspaceRootDir); + // Build a map of package paths to their versions for collision detection // Sort packages by key to ensure consistent order (just like the npm cli does it) const sortedExtractedPackages = Object.create(null); - const sortedKeys = Object.keys(extractedPackages).sort((a, b) => a.localeCompare(b)); + const sortedKeys = Array.from(physicalTree.keys()).sort((a, b) => a.localeCompare(b)); for (const key of sortedKeys) { - sortedExtractedPackages[key] = extractedPackages[key]; + sortedExtractedPackages[key] = physicalTree.get(key)[0]; } // Generate npm-shrinkwrap.json const shrinkwrap = { name: targetPackageName, - version: cliNode.version, + version: targetNode.version, lockfileVersion: 3, requires: true, packages: sortedExtractedPackages }; + // Validate the generated shrinkwrap using Arborist + await validateLockfile(shrinkwrap, targetNode); + return shrinkwrap; } -/** - * Normalize package locations from workspace-specific paths to standard npm paths. - * Examples (assuming @ui5/cli is the targetPackageName): - * - packages/cli/node_modules/foo -> node_modules/foo - * - packages/fs/node_modules/bar -> node_modules/@ui5/fs/node_modules/bar - * - * @param {string} location - Package location from arborist - * @param {object} node - Package node from arborist - * @param {string} targetPackageName - Target package name for shrinkwrap file - * @param {string} rootPackageName - Root / workspace package name - * @returns {string} - Normalized location for npm-shrinkwrap.json - */ -function normalizePackageLocation(location, node, targetPackageName, rootPackageName) { - const topPackageName = node.top.packageName; - if (topPackageName === targetPackageName) { - // Remove location for packages within target package (e.g. @ui5/cli) - return location.substring(node.top.location.length + 1); - } else if (topPackageName !== rootPackageName) { - // Add package within node_modules of actual package name (e.g. @ui5/fs) - return `node_modules/${topPackageName}/${location.substring(node.top.location.length + 1)}`; +function resolveVirtualTree(node, virtualFlatTree, curPath, parentNode) { + if (node.isLink) { + node = node.target; } - // If it's already within the root workspace package, keep as-is - return location; -} -function collectDependencies(node, relevantPackageLocations) { - if (relevantPackageLocations.has(node.location)) { - // Already processed + const fullPath = [curPath, node.name].join(" | "); + + if (virtualFlatTree.some(([path]) => path === fullPath)) { return; } - relevantPackageLocations.set(node.location, node); + if (node.isLink) { node = node.target; } + + virtualFlatTree.push([fullPath, [node, parentNode]]); + for (const edge of node.edgesOut.values()) { if (edge.dev || !edge.to) { // Skip dev dependencies and optional peer dependencies that are not installed continue; } - collectDependencies(edge.to, relevantPackageLocations); + + resolveVirtualTree(edge.to, virtualFlatTree, fullPath, node); } } +async function buildPhysicalTree( + virtualFlatTree, physicalTree, packageLockJson, workspaceRootDir) { + // Sort by path depth and then alphabetically to ensure parent + // packages are processed before children. It's important to + // process parents first to correctly handle version collisions and hoisting + virtualFlatTree.sort(([pathA], [pathB]) => { + if (pathA.split(" | ").length < pathB.split(" | ").length) { + return -1; + } else if (pathA.split(" | ").length > pathB.split(" | ").length) { + return 1; + } else { + return pathA.localeCompare(pathB); + } + }); + const targetNode = virtualFlatTree[0][1][0]; + const targetPackageName = targetNode.packageName; + + // Collect information to resolve potential version conflicts later + const statsToResolveConflicts = new Map(); + for (const [, nodes] of virtualFlatTree) { + const packageLoc = resolveLocation(nodes, physicalTree, targetPackageName); + const [node, parentNode] = nodes; + const {version} = node; + const isTargetPackageHardDep = (parentNode?.packageName === targetPackageName); + + // index 0: Set of versions found for this location + // index 1: Map of version -> count + // (this will be used eventually to elect the most common version in root node_modules) + // index 2: If target package has direct dependency here, the version + const packageStats = statsToResolveConflicts.get(packageLoc) || [new Set(), Object.create(null)]; + packageStats[0].add(version); + packageStats[1][version] ??= 0; + packageStats[1][version]++; + if (isTargetPackageHardDep) { + if (packageStats[2]) { + throw new Error(`Impossible to resolve hoisting conflicts. ` + + `Target package direct dependency "${node.packageName}" ` + + `has multiple versions: ${packageStats[2]} and ${version}.`); + } + packageStats[2] = version; + } + + statsToResolveConflicts.set(packageLoc, packageStats); + } + + const resolvedPackageLocations = new Map(); + for (const [, nodes] of virtualFlatTree) { + let packageLoc = resolveLocation(nodes, physicalTree, targetPackageName); + const [node, parentNode] = nodes; + const {location, version} = node; + const pkg = packageLockJson.packages[location]; + + const isRootNodeModulesLocation = `node_modules/${node.packageName}` === packageLoc; + const isTargetModuleDependency = (parentNode?.packageName === targetPackageName); + + // Handle version conflicts in root node_modules + if (isRootNodeModulesLocation && !isTargetModuleDependency) { + const packageStats = statsToResolveConflicts.get(packageLoc); + const hasConflictingLocationAndVersion = packageStats[0].size > 1; + // Which is the version of the package that's (eventually) used as + // dependency of the target package. + let selectedVersionForRootNodeModules = version; + + if (hasConflictingLocationAndVersion) { + const targetPackageVersion = packageStats[2]; + const versionsCount = packageStats[1]; + // Use target package direct dependency version if available, + // otherwise elect the most common version among dependents + selectedVersionForRootNodeModules = targetPackageVersion ?? + Object.keys(packageStats[1]).reduce((acc, versionKey) => { + return versionsCount[acc] > versionsCount[versionKey] ? acc : versionKey; + }); + } + + if (selectedVersionForRootNodeModules !== version) { + const parentPath = resolvedPackageLocations.get(parentNode) ?? + // Fallback in case parentNode is not yet resolved (should never happen) + // check virtualFlatTree.sort(...) above + normalizePackageLocation(parentNode.location, parentNode, targetPackageName); + packageLoc = parentPath ? `${parentPath}/${packageLoc}` : packageLoc; + } + } + + if (packageLoc !== "" && !pkg.resolved) { + // For all but the root package, ensure that "resolved" and "integrity" fields are present + // These are always missing for locally linked packages, but sometimes also for others (e.g. if installed + // from local cache) + const {resolved, integrity} = + await fetchPackageMetadata(node.packageName, node.version, workspaceRootDir); + pkg.resolved = resolved; + pkg.integrity = integrity; + } + + resolvedPackageLocations.set(node, packageLoc); + physicalTree.set(packageLoc, [pkg, node]); + } +} + +function resolveLocation(nodes, physicalTree, targetPackageName) { + let packageLoc; + const [node, parentNode] = nodes; + const {location} = node; + + if (node.packageName === targetPackageName) { + // Make the target package the root package + packageLoc = ""; + if (physicalTree[location]) { + throw new Error(`Duplicate root package entry for "${targetPackageName}"`); + } + } else if (parentNode?.packageName === targetPackageName) { + // Direct dependencies of the target package go into node_modules. + packageLoc = `node_modules/${node.packageName}`; + } else { + packageLoc = normalizePackageLocation(location, node, targetPackageName); + } + + return packageLoc; +} + +function normalizePackageLocation(location, node, targetPackageName) { + const topPackageName = node.top.packageName; + const rootPackageName = node.root.packageName; + let curLocation = location; + if (topPackageName === targetPackageName) { + // Remove location for packages within target package (e.g. @ui5/cli) + curLocation = location.substring(node.top.location.length + 1); + } else if (topPackageName !== rootPackageName) { + // Add package within node_modules of actual package name (e.g. @ui5/fs) + curLocation = `node_modules/${topPackageName}/${location.substring(node.top.location.length + 1)}`; + } + // If it's already within the root workspace package, keep as-is + return curLocation.endsWith("/") ? curLocation.slice(0, -1) : curLocation; +} + /** * Fetch package metadata from npm registry using pacote * @@ -184,3 +284,57 @@ async function fetchPackageMetadata(packageName, version, workspaceRoot) { throw new Error(`Could not fetch registry metadata for ${packageName}@${version}: ${errorMessage}`); } } + +/** + * Validate the generated shrinkwrap structure using Arborist + * This ensures the shrinkwrap is well-formed and can be loaded by npm/Arborist + * + * @param {object} shrinkwrap - Generated shrinkwrap object + * @param {object} targetNode - Target package node from Arborist + * @throws {Error} If the shrinkwrap is invalid or cannot be loaded by Arborist + */ +async function validateLockfile(shrinkwrap, targetNode) { + let tempDir; + try { + // Create a temporary directory for validation + tempDir = await mkdtemp(path.join(tmpdir(), "shrinkwrap-validation-")); + + // Create package.json (production dependencies only) + const packageJson = { + name: shrinkwrap.name, + version: shrinkwrap.version, + dependencies: targetNode.package.dependencies || {} + }; + + // Write package.json and npm-shrinkwrap.json + await writeFile( + path.join(tempDir, "package.json"), + JSON.stringify(packageJson, null, 2) + ); + await writeFile( + path.join(tempDir, "npm-shrinkwrap.json"), + JSON.stringify(shrinkwrap, null, 2) + ); + + // Attempt to load the virtual tree with Arborist + // This validates that the shrinkwrap is well-formed + const arb = new Arborist({ + path: tempDir, + }); + + await arb.loadVirtual(); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error( + `Generated shrinkwrap validation failed: ${errorMessage}. ` + + `The shrinkwrap structure is invalid or cannot be loaded by Arborist.` + ); + } finally { + // Clean up temporary directory + if (tempDir) { + await rm(tempDir, {recursive: true, force: true}).catch(() => { + // Ignore cleanup errors + }); + } + } +} diff --git a/internal/shrinkwrap-extractor/package.json b/internal/shrinkwrap-extractor/package.json index 311864ef04b..7e15a1d2db8 100644 --- a/internal/shrinkwrap-extractor/package.json +++ b/internal/shrinkwrap-extractor/package.json @@ -23,6 +23,7 @@ "unit": "node --test test/lib/convertToShrinkwrap.js", "unit-watch": "node --test --watch test/lib/convertToShrinkwrap.js", "coverage": "node --test --experimental-test-coverage 'test/lib/convertToShrinkwrap.js'", + "integration": "node --test test/integration/shrinkwrap-validation.js", "lint": "eslint ." }, "keywords": [ diff --git a/internal/shrinkwrap-extractor/test/expected/package.b/npm-shrinkwrap.json b/internal/shrinkwrap-extractor/test/expected/package.b/npm-shrinkwrap.json index 3e91f55dab1..650c1d1b456 100644 --- a/internal/shrinkwrap-extractor/test/expected/package.b/npm-shrinkwrap.json +++ b/internal/shrinkwrap-extractor/test/expected/package.b/npm-shrinkwrap.json @@ -1043,6 +1043,47 @@ "resolved": "https://registry.npmjs.org/package/version.tgz", "integrity": "sha512-mock-integrity-hash" }, + "node_modules/@ui5/cli/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@ui5/cli/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@ui5/cli/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@ui5/fs": { "name": "@ui5/fs", "version": "4.0.2", @@ -1346,28 +1387,28 @@ } }, "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "version": "6.2.2", "license": "MIT", "engines": { - "node": ">=8" - } + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + }, + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" }, "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "version": "6.2.3", "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, "engines": { - "node": ">=8" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } + }, + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" }, "node_modules/argparse": { "version": "2.0.1", @@ -6044,21 +6085,21 @@ "integrity": "sha512-mock-integrity-hash" }, "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "version": "8.1.0", "license": "MIT", "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" }, "engines": { - "node": ">=10" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } + }, + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" }, "node_modules/wrap-ansi-cjs": { "name": "wrap-ansi", diff --git a/internal/shrinkwrap-extractor/test/expected/package.c/npm-shrinkwrap.json b/internal/shrinkwrap-extractor/test/expected/package.c/npm-shrinkwrap.json new file mode 100644 index 00000000000..85ff3e55e92 --- /dev/null +++ b/internal/shrinkwrap-extractor/test/expected/package.c/npm-shrinkwrap.json @@ -0,0 +1,57 @@ +{ + "name": "@ui5/target", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@ui5/target", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@sapui5/some-thirdparty": "^2.0.0", + "@ui5/module-a": "^1.0.0", + "@ui5/module-b": "^1.0.0" + }, + "devDependencies": {} + }, + "node_modules/@sapui5/some-thirdparty": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" + }, + "node_modules/@ui5/module-a/node_modules/@sapui5/some-thirdparty": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" + }, + "node_modules/@ui5/module-b/node_modules/@sapui5/some-thirdparty": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" + }, + "node_modules/@ui5/module-a": { + "name": "@ui5/module-a", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@ui5/module-b": "^1.0.0", + "@sapui5/some-thirdparty": "^1.0.0" + }, + "devDependencies": {}, + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" + }, + "node_modules/@ui5/module-b": { + "name": "@ui5/module-b", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@sapui5/some-thirdparty": "^1.0.0" + }, + "devDependencies": {}, + "resolved": "https://registry.npmjs.org/package/version.tgz", + "integrity": "sha512-mock-integrity-hash" + } + } +} \ No newline at end of file diff --git a/internal/shrinkwrap-extractor/test/fixture/project.c/package-lock.fixture.json b/internal/shrinkwrap-extractor/test/fixture/project.c/package-lock.fixture.json new file mode 100644 index 00000000000..240b184baef --- /dev/null +++ b/internal/shrinkwrap-extractor/test/fixture/project.c/package-lock.fixture.json @@ -0,0 +1,68 @@ +{ + "name": "@ui5/cli-monorepo", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "monorepo-root", + "version": "0.0.1", + "license": "Apache-2.0", + "workspaces": [ + "packages/target", + "packages/module-a", + "packages/module-b" + ], + "dependencies": {}, + "devDependencies": {} + }, + "node_modules/@ui5/target": { + "resolved": "packages/target", + "link": true + }, + "node_modules/@ui5/module-a": { + "resolved": "packages/module-a", + "link": true + }, + "node_modules/@ui5/module-b": { + "resolved": "packages/module-b", + "link": true + }, + "packages/target": { + "name": "@ui5/target", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@sapui5/some-thirdparty": "^2.0.0", + "@ui5/module-a": "^1.0.0", + "@ui5/module-b": "^1.0.0" + }, + "devDependencies": {} + }, + "packages/module-a": { + "name": "@ui5/module-a", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@ui5/module-b": "^1.0.0", + "@sapui5/some-thirdparty": "^1.0.0" + }, + "devDependencies": {} + }, + "packages/module-b": { + "name": "@ui5/module-b", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@sapui5/some-thirdparty": "^1.0.0" + }, + "devDependencies": {} + }, + "node_modules/@sapui5/some-thirdparty": { + "version": "1.0.0" + }, + "packages/target/node_modules/@sapui5/some-thirdparty": { + "version": "2.0.0" + } + } +} diff --git a/internal/shrinkwrap-extractor/test/integration/shrinkwrap-validation.js b/internal/shrinkwrap-extractor/test/integration/shrinkwrap-validation.js new file mode 100644 index 00000000000..6f2d846a5fa --- /dev/null +++ b/internal/shrinkwrap-extractor/test/integration/shrinkwrap-validation.js @@ -0,0 +1,220 @@ +import path from "node:path"; +import { + writeFile, + unlink, + symlink, + rm, + mkdir, + stat, + readdir, + readFile, +} from "node:fs/promises"; +import {execFile} from "node:child_process"; +import {promisify} from "node:util"; +import convertPackageLockToShrinkwrap from "../../lib/convertPackageLockToShrinkwrap.js"; +import {test} from "node:test"; +import assert from "node:assert"; + +const execFileAsync = promisify(execFile); + +/** + * Create a temporary symlink from package-lock.fixture.json to package-lock.json + * This is needed because @npmcli/arborist.loadVirtual() expects package-lock.json + * + * @param {string} fixtureDir - Directory containing the fixture file + * @returns {Promise} Path to the created symlink + */ +async function setupFixtureSymlink(fixtureDir) { + const symlinkPath = path.join(fixtureDir, "package-lock.json"); + const targetPath = "package-lock.fixture.json"; + await symlink(targetPath, symlinkPath); + return symlinkPath; +} + +async function setupTestEnvironment(testDirName) { + const __dirname = import.meta.dirname; + const fixtureDir = path.join(__dirname, "..", "fixture", "project.a"); + const symlinkPath = await setupFixtureSymlink(fixtureDir); + + // Generate shrinkwrap with real npm registry data + const shrinkwrapJson = await convertPackageLockToShrinkwrap(fixtureDir, "@ui5/cli"); + + // Create test directory + const testDir = path.join(__dirname, "..", "tmp", testDirName); + await rm(testDir, {recursive: true, force: true}); + await mkdir(testDir, {recursive: true}); + + // Create package.json from shrinkwrap root package data + const rootPackage = shrinkwrapJson.packages[""]; + const packageJson = { + name: shrinkwrapJson.name, + version: shrinkwrapJson.version, + dependencies: rootPackage.dependencies || {}, + engines: rootPackage.engines, + bin: rootPackage.bin, + }; + + // Write package.json and npm-shrinkwrap.json + await writeFile( + path.join(testDir, "package.json"), + JSON.stringify(packageJson, null, 2) + ); + await writeFile( + path.join(testDir, "npm-shrinkwrap.json"), + JSON.stringify(shrinkwrapJson, null, 2) + ); + + // Cleanup function + const cleanup = async () => { + await rm(testDir, {recursive: true, force: true}).catch(() => {}); + await unlink(symlinkPath).catch(() => {}); + }; + + return {shrinkwrapJson, testDir, cleanup}; +} + +// Recursively get all installed packages from node_modules +async function getInstalledPackages(dir, prefix = "") { + const packages = []; + const entries = await readdir(dir, {withFileTypes: true}); + + for (const entry of entries) { + if (!entry.isDirectory() || entry.name === ".bin") continue; + + // Check if it's a scoped package directory + if (entry.name.startsWith("@")) { + const scopedPackages = await getInstalledPackages( + path.join(dir, entry.name), + prefix ? `${prefix}/${entry.name}` : entry.name + ); + packages.push(...scopedPackages); + } else { + // Regular package + const pkgName = prefix ? `${prefix}/${entry.name}` : entry.name; + packages.push(pkgName); + + // Check for nested node_modules + const nestedNodeModules = path.join(dir, entry.name, "node_modules"); + try { + const nestedStat = await stat(nestedNodeModules); + if (nestedStat.isDirectory()) { + const nestedPackages = await getInstalledPackages( + nestedNodeModules, + `${pkgName}/node_modules` + ); + packages.push(...nestedPackages); + } + } catch { + // No nested node_modules, that's fine + } + } + } + return packages; +} + +test("Integration: Generated shrinkwrap can be used with npm ci", async (t) => { + const {testDir, cleanup} = await setupTestEnvironment("npm-ci-test"); + t.after(cleanup); + + // Run npm ci + const {stderr} = await execFileAsync("npm", ["ci"], { + cwd: testDir, + env: {...process.env, NO_UPDATE_NOTIFIER: "1"}, + }); + + if (stderr) { + console.warn("npm ci warnings:", stderr); + } + + // Verify node_modules was created + const nodeModulesPath = path.join(testDir, "node_modules"); + const nodeModulesStat = await stat(nodeModulesPath); + assert.ok(nodeModulesStat.isDirectory(), "node_modules should be created"); + + // Verify key packages are installed + const keyPackages = ["@ui5/builder", "chalk", "yargs"]; + for (const pkg of keyPackages) { + const pkgStat = await stat(path.join(nodeModulesPath, pkg)).catch(() => null); + assert.ok(pkgStat?.isDirectory(), `Package ${pkg} should be installed`); + } +}); + +test("Integration: Verify dependency tree with npm ls", async (t) => { + const {shrinkwrapJson, testDir, cleanup} = await setupTestEnvironment("npm-ls-test"); + t.after(cleanup); + + // Run npm ci first + await execFileAsync("npm", ["ci"], { + cwd: testDir, + env: {...process.env, NO_UPDATE_NOTIFIER: "1"}, + }); + + // Run npm ls to verify the dependency tree + const {stdout} = await execFileAsync("npm", ["ls", "--all", "--json"], { + cwd: testDir, + env: {...process.env, NO_UPDATE_NOTIFIER: "1"}, + }); + + const dependencyTree = JSON.parse(stdout); + + // Verify the root package + assert.equal(dependencyTree.name, "@ui5/cli"); + assert.equal(dependencyTree.version, shrinkwrapJson.version); + + // Verify key dependencies are present + assert.ok(dependencyTree.dependencies, "Should have dependencies"); + const deps = dependencyTree.dependencies; + + // Check for key packages + assert.ok(deps["@ui5/builder"], "Should have @ui5/builder"); + assert.ok(deps.chalk, "Should have chalk"); + assert.ok(deps.yargs, "Should have yargs"); + + // Verify no devDependencies leaked in + assert.ok(!deps.eslint, "Should not have eslint (devDependency)"); + assert.ok(!deps.ava, "Should not have ava (devDependency)"); +}); + +test("Integration: Verify installed packages match shrinkwrap exactly", async (t) => { + const {shrinkwrapJson, testDir, cleanup} = await setupTestEnvironment("npm-match-test"); + t.after(cleanup); + + // Run npm ci + await execFileAsync("npm", ["ci"], { + cwd: testDir, + env: {...process.env, NO_UPDATE_NOTIFIER: "1"}, + }); + + const nodeModulesPath = path.join(testDir, "node_modules"); + + // Get all packages from shrinkwrap (excluding root "") + const shrinkwrapPackages = Object.keys(shrinkwrapJson.packages) + .filter((key) => key && key.startsWith("node_modules/")) + .map((key) => key.replace("node_modules/", "")); + + const installedPackages = await getInstalledPackages(nodeModulesPath); + + // Check no extra packages are installed + const shrinkwrapSet = new Set(shrinkwrapPackages); + const installedSet = new Set(installedPackages); + + const extraPackages = installedPackages.filter((pkg) => !shrinkwrapSet.has(pkg)); + assert.equal(extraPackages.length, 0, + `Found ${extraPackages.length} extra packages not in shrinkwrap: ` + + `${extraPackages.slice(0, 5).join(", ")}${extraPackages.length > 5 ? "..." : ""}`); + + const missingPackages = shrinkwrapPackages.filter((pkg) => !installedSet.has(pkg)); + assert.equal(missingPackages.length, 0, + `Missing ${missingPackages.length} packages from shrinkwrap: ` + + `${missingPackages.slice(0, 5).join(", ")}${missingPackages.length > 5 ? "..." : ""}`); + + // Verify versions match for all packages + for (const pkg of shrinkwrapPackages) { + const shrinkwrapVersion = shrinkwrapJson.packages[`node_modules/${pkg}`].version; + const pkgJsonPath = path.join(nodeModulesPath, pkg, "package.json"); + + const pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf8")); + assert.equal(pkgJson.version, shrinkwrapVersion, + `Version mismatch for ${pkg}: shrinkwrap=${shrinkwrapVersion}, installed=${pkgJson.version}`); + } +}); diff --git a/internal/shrinkwrap-extractor/test/lib/convertToShrinkwrap.js b/internal/shrinkwrap-extractor/test/lib/convertToShrinkwrap.js index 2f9ea90d476..ea0e32bea96 100644 --- a/internal/shrinkwrap-extractor/test/lib/convertToShrinkwrap.js +++ b/internal/shrinkwrap-extractor/test/lib/convertToShrinkwrap.js @@ -86,40 +86,57 @@ test("Convert package-lock.json to shrinkwrap", async (t) => { test("Workspace paths should be normalized to node_modules format", async (t) => { const __dirname = import.meta.dirname; - const cwd = path.join(__dirname, "..", "fixture", "project.a"); const symlinkPath = await setupFixtureSymlink(cwd); t.after(async () => await unlink(symlinkPath).catch(() => {})); - const targetPackageName = "@ui5/cli"; - const shrinkwrapJson = await convertPackageLockToShrinkwrap(cwd, targetPackageName); - - // Verify that no package paths contain workspace prefixes like "packages/cli/node_modules/..." - const packagePaths = Object.keys(shrinkwrapJson.packages); + const shrinkwrapJson = await convertPackageLockToShrinkwrap(cwd, "@ui5/cli"); + const packagePaths = Object.keys(shrinkwrapJson.packages).filter((p) => p !== ""); + // All paths must start with node_modules/, never with packages/ for (const packagePath of packagePaths) { - // Skip root package (empty string) - if (packagePath === "") continue; - - // Assert that no path starts with "packages/" assert.ok(!packagePath.startsWith("packages/"), - `Package path "${packagePath}" should not start with "packages/" prefix`); - - // Assert that non-root paths start with "node_modules/" + `Path "${packagePath}" should not contain workspace prefix`); assert.ok(packagePath.startsWith("node_modules/"), - `Package path "${packagePath}" should start with "node_modules/" prefix`); + `Path "${packagePath}" should start with node_modules/`); } - // Specifically check a package that would have been under packages/cli/node_modules in the monorepo - // The "@npmcli/config" package is a direct dependency that exists in the CLI's node_modules - const npmCliConfigPackage = shrinkwrapJson.packages["node_modules/@npmcli/config"]; - assert.ok(npmCliConfigPackage, "The '@npmcli/config' package should be present at normalized path"); - assert.equal(npmCliConfigPackage.version, "9.0.0", "@npmcli/config package should have correct version"); + // Verify a CLI dependency was normalized correctly + const npmCliConfig = shrinkwrapJson.packages["node_modules/@npmcli/config"]; + assert.ok(npmCliConfig, "@npmcli/config should be at normalized path"); + assert.equal(npmCliConfig.version, "9.0.0"); + + console.log(`✓ All ${packagePaths.length} package paths correctly normalized`); +}); + +test("Version collisions: root packages get priority at top level", async (t) => { + const __dirname = import.meta.dirname; + const cwd = path.join(__dirname, "..", "fixture", "project.b"); + const symlinkPath = await setupFixtureSymlink(cwd); + t.after(async () => await unlink(symlinkPath).catch(() => {})); + + const shrinkwrapJson = await convertPackageLockToShrinkwrap(cwd, "@ui5/cli"); + + // ansi-regex: root has v6.2.2, CLI workspace has v5.0.1, but not direct dependency to @ui5/cli + const rootAnsiRegex = shrinkwrapJson.packages["node_modules/ansi-regex"]; + assert.equal(rootAnsiRegex?.version, "6.2.2", "Root ansi-regex at top level"); + + Object.keys(shrinkwrapJson.packages) + .filter((pkg) => pkg.endsWith("node_modules/ansi-regex") && pkg !== "node_modules/ansi-regex") + .forEach((pkgName) => { + assert.equal(shrinkwrapJson.packages[pkgName]?.version, + "5.0.1", `Workspace ansi-regex nested under @ui5/cli -> ${pkgName}`); + }); - console.log(`✓ All ${packagePaths.length - 1} package paths correctly normalized`); + // Verify root version satisfies dependents + const stripAnsi = shrinkwrapJson.packages["node_modules/strip-ansi"]; + assert.equal(stripAnsi.dependencies["ansi-regex"], "^6.0.1"); + assert.ok(rootAnsiRegex.version.startsWith("6."), "Root v6.2.2 satisfies ^6.0.1"); + + console.log("✓ Root package prioritized at top level, workspace version nested"); }); -test("Compare generated shrinkwrap with expected result", async (t) => { +test("Compare generated shrinkwrap with expected result: package.a", async (t) => { // Setup mock to prevent actual npm registry requests const mockRestore = setupPacoteMock(); t.after(() => mockRestore()); @@ -172,8 +189,7 @@ test("Compare generated shrinkwrap with expected result", async (t) => { "Generated shrinkwrap packages should match expected"); }); - -test("Compare generated shrinkwrap with expected result", async (t) => { +test("Compare generated shrinkwrap with expected result: package.b", async (t) => { // Setup mock to prevent actual npm registry requests const mockRestore = setupPacoteMock(); t.after(() => mockRestore()); @@ -204,6 +220,37 @@ test("Compare generated shrinkwrap with expected result", async (t) => { "Generated shrinkwrap packages should match expected"); }); +test("Compare generated shrinkwrap with expected result: package.c", async (t) => { + // Setup mock to prevent actual npm registry requests + const mockRestore = setupPacoteMock(); + t.after(() => mockRestore()); + + const __dirname = import.meta.dirname; + const generatedShrinkwrapPath = path.join(__dirname, "..", "tmp", "package.c", "npm-shrinkwrap.generated.json"); + // Clean any existing generated file + await mkdir(path.dirname(generatedShrinkwrapPath), {recursive: true}); + await unlink(generatedShrinkwrapPath).catch(() => {}); + + // Generate shrinkwrap from fixture + const cwd = path.join(__dirname, "..", "fixture", "project.c"); + const symlinkPath = await setupFixtureSymlink(cwd); + t.after(async () => await unlink(symlinkPath).catch(() => {})); + + const targetPackageName = "@ui5/target"; + + const generatedShrinkwrap = await convertPackageLockToShrinkwrap(cwd, targetPackageName); + + // Load expected shrinkwrap + const expectedShrinkwrapPath = path.join(__dirname, "..", "expected", "package.c", "npm-shrinkwrap.json"); + const expectedShrinkwrap = await readJson(expectedShrinkwrapPath); + + // Write generated shrinkwrap to tmp dir for debugging purposes + await writeFile(generatedShrinkwrapPath, JSON.stringify(generatedShrinkwrap, null, "\t"), "utf-8"); + + assert.deepEqual(generatedShrinkwrap.packages, expectedShrinkwrap.packages, + "Generated shrinkwrap packages should match expected"); +}); + test("Optional peer dependencies with null edges should be excluded", async (t) => { // Guards against: ws declares bufferutil and utf-8-validate as peerOptional, but they are not // installed. Arborist represents these as edges with edge.to === null. The generator must skip @@ -315,6 +362,38 @@ test("Error handling - invalid package-lock.json files", async (t) => { ); }); +test("Arborist validation catches invalid shrinkwrap structure", async (t) => { + const __dirname = import.meta.dirname; + const {Arborist} = await import("@npmcli/arborist"); + + // Mock Arborist.loadVirtual to simulate validation failure + const originalLoadVirtual = Arborist.prototype.loadVirtual; + let callCount = 0; + + Arborist.prototype.loadVirtual = async function(...args) { + callCount++; + // First call is the real load (in convertPackageLockToShrinkwrap) + // Second call is the validation + if (callCount === 2) { + throw new Error("Mock validation error: Invalid dependency tree"); + } + return originalLoadVirtual.apply(this, args); + }; + + const cwd = path.join(__dirname, "..", "fixture", "project.a"); + const symlinkPath = await setupFixtureSymlink(cwd); + + t.after(async () => { + Arborist.prototype.loadVirtual = originalLoadVirtual; + await unlink(symlinkPath).catch(() => {}); + }); + + await assert.rejects( + convertPackageLockToShrinkwrap(cwd, "@ui5/cli"), + /Generated shrinkwrap validation failed.*Mock validation error/ + ); +}); + async function readJson(filePath) { const jsonString = await readFile(filePath, {encoding: "utf-8"}); return JSON.parse(jsonString);