From 164dfb591e1a47999a044417f8944eca4d38614a Mon Sep 17 00:00:00 2001 From: Rob Hogan <2590098+robhogan@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:32:05 +0000 Subject: [PATCH] metro-file-map: Lazily stat files in the node crawler Without Watchman, the node crawler `lstat`s every file on every start, so that `getDifference` can tell what changed while Metro wasn't running: https://github.com/react/metro/blob/56efab302200568c919ac1b3c28dacf8d84fc692/packages/metro-file-map/src/crawlers/node/index.js#L83-L96 Most of those files are never read, and an mtime only matters for a file with something derived from it (a SHA-1 or plugin data) - that's all a change can invalidate. This takes the idea in @kitten's #1686 and makes it hold across restarts. A regular file is crawled without a stat, and its mtime is recorded when data is first derived from it, from a stat taken before it's read. A file with no mtime then has nothing that could be stale, so a start only stats the files that had an mtime before. - A recrawl while watching still stats every file, because consumers like `PackageCache` hold contents read outside the file map. - Symlinks are still read eagerly - every lookup through a link needs its target. - `linkStats()` gives a null `modifiedTime` and `size` for a file that hasn't been stat'd. - `getDifference()` also returns the files it can't compare without a stat (`filesToStat`), so the crawler stats only those. - `CACHE_BREAKER` is bumped, or an existing cache would keep every file stat'd. ## Benchmark **Without Watchman, this takes 43-50% off a cold or warm start on Linux and Windows** (Linux warm: 635ms -> 360ms), and 25-27% off on macOS. Watch mode gains less on Linux and Windows, where setting up the watcher dominates - the rest of this stack deals with that. The cost is a stat when a file is first read: 9-30% more on a first SHA-1 pass over the bundle's files (6-46ms here), once per file per cache. | | `main` | This PR | vs `main` | |---|---|---|---| | macOS: Cold start | 127ms | 93ms | -26.7% (-35.5% to -19.2%) | | Linux: Cold start | 645ms | 331ms | -48.8% (-49.6% to -48.0%) | | Windows: Cold start | 361ms | 182ms | -49.7% (-51.1% to -49.2%) | | macOS: Warm start | 119ms | 90ms | -24.8% (-26.1% to -22.7%) | | Linux: Warm start | 635ms | 360ms | -43.3% (-44.1% to -42.6%) | | Windows: Warm start | 354ms | 203ms | -42.6% (-43.0% to -42.1%) | | macOS: Watch-mode start, warm cache | 130ms | 104ms | -20.0% (-21.2% to -18.2%) | | Linux: Watch-mode start, warm cache | 1,887ms | 1,614ms | -14.5% (-15.4% to -13.7%) | | Windows: Watch-mode start, warm cache | 2,577ms | 2,417ms | -6.2% (-6.7% to -5.8%) | | macOS: First SHA-1 of the bundle's 3,125 files | 68ms | 74ms | +8.8% (+7.7% to +10.6%) | | Linux: First SHA-1 of the bundle's 3,125 files | 100ms | 130ms | +29.6% (+24.1% to +37.2%) | | Windows: First SHA-1 of the bundle's 3,125 files | 258ms | 304ms | +17.7% (+14.9% to +19.2%) | ## Changelog ``` - **[Performance]**: metro-file-map: Start faster without Watchman, by not stat-ing files whose contents Metro hasn't read ``` ## Test plan New unit tests, and a `stats-between-starts-test.js` that edits files between two runs of FileMap over a real directory and cache. SHA-1s, eager and lazy plugin data and a retargeted symlink are recomputed, and an unread file is stat'd only once it's read. These include the cases #1686 gets wrong. Benchmark app (AI-driven harness, used throughout this stack): a React Native 0.87.1 app with 23 common dependencies, 34k files in the file map and 3.1k modules in its iOS dev bundle, started through Metro's `DependencyGraph` with `resolver.useWatchman: false`. Each round runs, per variant: 2 cold starts, a watch-mode session that computes the SHA-1s of the bundle's files (as transforming them does) and saves them to the cache, 3 warm starts and 3 watch-mode starts from that cache, and 2 watch-mode cold starts. 10 interleaved rounds per OS, alternating the variant order, then medians and bootstrap 95% CIs. Heap is measured after a forced GC. Machines: Mac mini M4 Pro (macOS, APFS, Node 26.11.1), Mac mini i5-4278U (Linux Mint 22.3, ext4, Node 24.21.0) and a Ryzen 7 7730U laptop (Windows 11, NTFS, Defender on, Node 24.21.0). Measured on 1461d81e97fa (`main` on 2026-11-10), with this stack on top. --- packages/metro-file-map/API.md | 6 +- .../src/__tests__/addRoot-test.js | 55 ++-- .../src/__tests__/index-test.js | 6 + .../__tests__/stats-between-starts-test.js | 236 ++++++++++++++++++ .../src/__tests__/worker-test.js | 50 ++++ .../crawlers/__tests__/integration-test.js | 58 +++-- .../src/crawlers/__tests__/node-test.js | 129 +++++++++- .../metro-file-map/src/crawlers/node/index.js | 76 +++++- packages/metro-file-map/src/flow-types.js | 18 +- packages/metro-file-map/src/index.js | 2 +- .../metro-file-map/src/lib/FileProcessor.js | 14 ++ packages/metro-file-map/src/lib/TreeFS.js | 22 +- .../src/lib/__tests__/FileProcessor-test.js | 28 +++ .../src/lib/__tests__/TreeFS-test.js | 36 +++ packages/metro-file-map/src/worker.js | 15 +- 15 files changed, 688 insertions(+), 63 deletions(-) create mode 100644 packages/metro-file-map/src/__tests__/stats-between-starts-test.js diff --git a/packages/metro-file-map/API.md b/packages/metro-file-map/API.md index 44a0f989d2..87af82ff8c 100644 --- a/packages/metro-file-map/API.md +++ b/packages/metro-file-map/API.md @@ -176,7 +176,11 @@ interface FileSystem_2 { options?: Readonly<{ subpath?: string | undefined; }>, - ): {changedFiles: FileData; removedFiles: Set}; + ): { + changedFiles: FileData; + removedFiles: Set; + filesToStat: Map; + }; getOrComputeSha1(file: Path): Promise; getSerializableSnapshot(): CacheData['fileSystemData']; getSha1(file: Path): null | undefined | string; diff --git a/packages/metro-file-map/src/__tests__/addRoot-test.js b/packages/metro-file-map/src/__tests__/addRoot-test.js index 2f52b416d4..2efe191449 100644 --- a/packages/metro-file-map/src/__tests__/addRoot-test.js +++ b/packages/metro-file-map/src/__tests__/addRoot-test.js @@ -13,6 +13,7 @@ import type { CacheData, ChangeEvent, CrawlerOptions, + FileData, FileMetadata, WatcherBackendOptions, } from '../flow-types'; @@ -29,6 +30,16 @@ let mockFiles: Map; // Absolute path -> normal (rootDir-relative, POSIX) symlink target. let mockSymlinks: Map; +// What the node crawler returns of `getDifference`'s result. +const toCrawlResult = ({ + changedFiles, + removedFiles, +}: { + changedFiles: FileData, + removedFiles: Set, + ... +}) => ({changedFiles, removedFiles}); + const mockNodeCrawler = jest.fn((options: CrawlerOptions) => { const path = require('node:path'); const files = new Map(); @@ -69,9 +80,11 @@ const mockNodeCrawler = jest.fn((options: CrawlerOptions) => { } } return Promise.resolve( - options.previousState.fileSystem.getDifference(files, { - subpath: options.subpath, - }), + toCrawlResult( + options.previousState.fileSystem.getDifference(files, { + subpath: options.subpath, + }), + ), ); }); @@ -316,14 +329,16 @@ describe('FileMap.addRoot', () => { new Promise(resolve => { finishCrawl = () => resolve( - options.previousState.fileSystem.getDifference( - new Map([ - [ - path.relative(ROOT_DIR, EXTERNAL_FILE), - [1, 42, 0, null, 0], - ], - ]), - {subpath: options.subpath}, + toCrawlResult( + options.previousState.fileSystem.getDifference( + new Map([ + [ + path.relative(ROOT_DIR, EXTERNAL_FILE), + [1, 42, 0, null, 0], + ], + ]), + {subpath: options.subpath}, + ), ), ); resolveStarted(); @@ -376,14 +391,16 @@ describe('FileMap.addRoot', () => { // Like Watchman, list a root's files under its real path. mockNodeCrawler.mockImplementationOnce(options => Promise.resolve( - options.previousState.fileSystem.getDifference( - new Map([ - [ - path.relative(ROOT_DIR, path.join(SRC_DIR, 'index.js')), - [1, 42, 0, null, 0], - ], - [path.relative(ROOT_DIR, realFile), [1, 42, 0, null, 0]], - ]), + toCrawlResult( + options.previousState.fileSystem.getDifference( + new Map([ + [ + path.relative(ROOT_DIR, path.join(SRC_DIR, 'index.js')), + [1, 42, 0, null, 0], + ], + [path.relative(ROOT_DIR, realFile), [1, 42, 0, null, 0]], + ]), + ), ), ), ); diff --git a/packages/metro-file-map/src/__tests__/index-test.js b/packages/metro-file-map/src/__tests__/index-test.js index 652da71b5e..2048cfd728 100644 --- a/packages/metro-file-map/src/__tests__/index-test.js +++ b/packages/metro-file-map/src/__tests__/index-test.js @@ -252,6 +252,7 @@ const assertFileSystemEqual = (fileSystem: FileSystem, fileData: FileData) => { expect(fileSystem.getDifference(fileData)).toEqual({ changedFiles: new Map(), removedFiles: new Set(), + filesToStat: new Map(), }); }; @@ -1503,6 +1504,7 @@ describe('FileMap', () => { filePath: p('/project/fruits/Banana.js'), maybeReturnContent: false, pluginsToRun: [0], + returnStats: false, }, ], [ @@ -1511,6 +1513,7 @@ describe('FileMap', () => { filePath: p('/project/fruits/Pear.js'), maybeReturnContent: false, pluginsToRun: [0], + returnStats: false, }, ], [ @@ -1519,6 +1522,7 @@ describe('FileMap', () => { filePath: p('/project/fruits/Strawberry.js'), maybeReturnContent: false, pluginsToRun: [0], + returnStats: false, }, ], [ @@ -1527,6 +1531,7 @@ describe('FileMap', () => { filePath: p('/project/fruits/__mocks__/Pear.js'), maybeReturnContent: false, pluginsToRun: [0], + returnStats: false, }, ], [ @@ -1535,6 +1540,7 @@ describe('FileMap', () => { filePath: p('/project/vegetables/Melon.js'), maybeReturnContent: false, pluginsToRun: [0], + returnStats: false, }, ], ]); diff --git a/packages/metro-file-map/src/__tests__/stats-between-starts-test.js b/packages/metro-file-map/src/__tests__/stats-between-starts-test.js new file mode 100644 index 0000000000..945fea192f --- /dev/null +++ b/packages/metro-file-map/src/__tests__/stats-between-starts-test.js @@ -0,0 +1,236 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type {FileSystem} from '../flow-types'; + +import FileMap, {DiskCacheManager} from '../index'; +import FileDataPlugin from '../plugins/FileDataPlugin'; +import {createHash} from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +// The node crawler stats a regular file only once data is derived from its +// content, and treats one that was never stat'd as unchanged. These tests +// change files while no FileMap is running, as between two starts of Metro, +// against a real file system and cache. + +jest.useRealTimers(); + +type LengthPlugin = FileDataPlugin; + +let rootDir: string; +let cacheDirectory: string; + +beforeEach(() => { + rootDir = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'metro-file-map-root-')), + ); + cacheDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), 'metro-file-map-cache-'), + ); +}); + +afterEach(() => { + fs.rmSync(rootDir, {recursive: true, force: true}); + fs.rmSync(cacheDirectory, {recursive: true, force: true}); +}); + +const abs = (normalPath: string) => path.join(rootDir, normalPath); + +function write(normalPath: string, content: string) { + fs.mkdirSync(path.dirname(abs(normalPath)), {recursive: true}); + fs.writeFileSync(abs(normalPath), content); +} + +// Timestamps can be coarser than a test runs (about 15ms on Windows), so move +// the modified time on, as an edit some time after the last start would. +let offlineEdits = 0; +function editOffline(normalPath: string, content: string) { + write(normalPath, content); + const seconds = Date.now() / 1000 + 60 + ++offlineEdits; + fs.utimesSync(abs(normalPath), seconds, seconds); +} + +const sha1 = (content: string) => + createHash('sha1').update(content).digest('hex'); + +function createLengthPlugin(name: string, lazy: boolean): LengthPlugin { + return new FileDataPlugin({ + name, + cacheKey: name, + worker: { + modulePath: require.resolve('./lazy_plugin_worker.js'), + setupArgs: {}, + }, + filter: ({normalPath}) => normalPath.endsWith('.eager.js'), + lazy, + }); +} + +async function session( + fn: ( + fileSystem: FileSystem, + plugins: {eager: LengthPlugin, lazy: LengthPlugin}, + ) => Promise | void, +): Promise { + const eager = createLengthPlugin('eager', false); + const lazy = createLengthPlugin('lazy', true); + const fileMap = new FileMap({ + cacheManagerFactory: ({buildParameters}) => + new DiskCacheManager({buildParameters}, {cacheDirectory}), + enableSymlinks: true, + extensions: ['js'], + healthCheck: { + enabled: false, + interval: 10000, + timeout: 1000, + filePrefix: '.metro-file-map-health-check', + }, + maxWorkers: 1, + plugins: [eager, lazy], + retainAllFiles: true, + rootDir, + roots: [rootDir], + useWatchman: false, + watch: false, + }); + try { + const {fileSystem} = await fileMap.build(); + await fn(fileSystem, {eager, lazy}); + } finally { + await fileMap.end(); + } +} + +test('a SHA-1 is recomputed after an edit', async () => { + write('a.js', 'before'); + await session(async fileSystem => { + expect(await fileSystem.getOrComputeSha1(abs('a.js'))).toMatchObject({ + sha1: sha1('before'), + }); + }); + + editOffline('a.js', 'after'); + + await session(async fileSystem => { + expect(await fileSystem.getOrComputeSha1(abs('a.js'))).toMatchObject({ + sha1: sha1('after'), + }); + }); +}); + +test("an eager plugin's data is recomputed after an edit", async () => { + write('b.eager.js', 'before'); + await session((_, {eager}) => { + expect(eager.getOrComputeFileData(abs('b.eager.js'))).toMatchObject({ + data: {length: 6}, + }); + }); + + editOffline('b.eager.js', 'and after'); + + await session((_, {eager}) => { + expect(eager.getOrComputeFileData(abs('b.eager.js'))).toMatchObject({ + data: {length: 9}, + }); + }); +}); + +test("a lazy plugin's data is recomputed after an edit", async () => { + write('c.js', 'before'); + await session((_, {lazy}) => { + expect(lazy.getOrComputeFileData(abs('c.js'))).toMatchObject({ + data: {length: 6}, + }); + }); + + editOffline('c.js', 'and after'); + + await session((_, {lazy}) => { + expect(lazy.getOrComputeFileData(abs('c.js'))).toMatchObject({ + data: {length: 9}, + }); + }); +}); + +test('an edit to a file not yet read is seen when it is first read', async () => { + write('d.js', 'before'); + await session(() => {}); + + editOffline('d.js', 'and after'); + + await session(async (fileSystem, {lazy}) => { + expect(lazy.getOrComputeFileData(abs('d.js'))).toMatchObject({ + data: {length: 9}, + }); + expect(await fileSystem.getOrComputeSha1(abs('d.js'))).toMatchObject({ + sha1: sha1('and after'), + }); + }); +}); + +test('a retargeted symlink is followed to its new target', async () => { + write('pkgs/a@1/index.js', 'one'); + write('pkgs/a@2/index.js', 'two'); + fs.mkdirSync(abs('node_modules')); + const link = abs('node_modules/a'); + fs.symlinkSync(path.join('..', 'pkgs', 'a@1'), link, 'dir'); + + await session(async fileSystem => { + expect(fileSystem.lookup(abs('node_modules/a/index.js'))).toMatchObject({ + exists: true, + realPath: abs('pkgs/a@1/index.js'), + }); + await fileSystem.getOrComputeSha1(abs('node_modules/a/index.js')); + }); + + // As a package manager upgrading `a` + fs.unlinkSync(link); + fs.symlinkSync(path.join('..', 'pkgs', 'a@2'), link, 'dir'); + const seconds = Date.now() / 1000 + 60; + // $FlowFixMe[prop-missing] - lutimesSync is missing from flow-typed's node defs + fs.lutimesSync(link, seconds, seconds); + fs.rmSync(abs('pkgs/a@1'), {recursive: true}); + + await session(fileSystem => { + expect(fileSystem.lookup(abs('node_modules/a/index.js'))).toMatchObject({ + exists: true, + realPath: abs('pkgs/a@2/index.js'), + }); + }); +}); + +test('a start stats only the files with data derived from them', async () => { + for (let i = 0; i < 10; i++) { + write(`f${i}.js`, String(i)); + } + await session(async fileSystem => { + await fileSystem.getOrComputeSha1(abs('f0.js')); + }); + + const lstat = jest.spyOn(fs, 'lstat'); + try { + await session(fileSystem => { + // Without a stat, a file's size isn't known + expect(fileSystem.linkStats(abs('f1.js'))).toEqual({ + fileType: 'f', + modifiedTime: null, + size: null, + }); + }); + expect(lstat.mock.calls.map(([filePath]) => filePath)).toEqual([ + abs('f0.js'), + ]); + } finally { + lstat.mockRestore(); + } +}); diff --git a/packages/metro-file-map/src/__tests__/worker-test.js b/packages/metro-file-map/src/__tests__/worker-test.js index 19c7f8ec5c..8e57ac90da 100644 --- a/packages/metro-file-map/src/__tests__/worker-test.js +++ b/packages/metro-file-map/src/__tests__/worker-test.js @@ -48,6 +48,16 @@ jest.mock('node:fs', () => { return { ...jest.createMockFromModule('node:fs'), + statSync: jest.fn((filePath: string) => { + const entry = mockFs[filePath]; + if (entry == null) { + throw new Error(`Cannot stat path '${filePath}'.`); + } + return { + mtime: new Date(1700000000000), + size: typeof entry === 'string' ? entry.length : 8, + }; + }), readFileSync: jest.fn((path, options) => { const entry = mockFs[path]; if (entry) { @@ -71,6 +81,7 @@ const defaults: WorkerMessage = { filePath: path.join('/project', 'notexist.js'), maybeReturnContent: false, pluginsToRun: [], + returnStats: false, }; const defaultHasteConfig = { @@ -112,6 +123,7 @@ describe('worker', () => { filePath: path.join('/project', 'notexist.js'), maybeReturnContent: false, pluginsToRun: [], + returnStats: false, }; test('simply computes SHA-1s when requested (works well with binary data)', async () => { @@ -238,6 +250,44 @@ describe('worker', () => { }); }); + test('returns the stats of the content it reads, when asked', async () => { + expect( + await worker({ + ...defaults, + computeSha1: true, + filePath: path.join('/project', 'fruits', 'apple.png'), + returnStats: true, + }), + ).toEqual({ + pluginData: [], + sha1: '4caece539b039b16e16206ea2478f8c5ffb2ca05', + stats: {mtime: 1700000000000, size: 8}, + }); + }); + + test('returns stats for data derived without reading content', async () => { + expect( + await workerWithHaste({ + ...defaults, + filePath: path.join('/project', 'fruits', 'Pear.js'), + returnStats: true, + }), + ).toEqual({ + pluginData: ['Pear'], + stats: {mtime: 1700000000000, size: expect.any(Number)}, + }); + expect(fs.readFileSync).not.toHaveBeenCalled(); + }); + + test('does not stat a file unless asked', async () => { + await worker({ + ...defaults, + computeSha1: true, + filePath: path.join('/project', 'fruits', 'Banana.js'), + }); + expect(fs.statSync).not.toHaveBeenCalled(); + }); + test('can be loaded directly without transpilation', async () => { const code = await jest .requireActual('node:fs') diff --git a/packages/metro-file-map/src/crawlers/__tests__/integration-test.js b/packages/metro-file-map/src/crawlers/__tests__/integration-test.js index 58bfe529bc..f1530c9f92 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/integration-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/integration-test.js @@ -72,30 +72,38 @@ function oneOf(this: $FlowFixMe, actual: unknown, ...expectOneOf: unknown[]) { * https://fburl.com/gdoc/y8dn025u */ expect.extend({oneOf}); -const CASES = [ - [ - true, - new Map([ - ['foo.js', [expect.any(Number), 245, 0, null, 0]], - [join('directory', 'bar.js'), [expect.any(Number), 245, 0, null, 0]], - [ - 'link-to-directory', - [expect.any(Number), 9, 0, null, expect.oneOf(1, 'directory')], - ], - [ - 'link-to-foo.js', - [expect.any(Number), 6, 0, null, expect.oneOf(1, 'foo.js')], - ], - ]), - ], - [ - false, - new Map([ - [join('directory', 'bar.js'), [expect.any(Number), 245, 0, null, 0]], - ['foo.js', [expect.any(Number), 245, 0, null, 0]], - ]), - ], -]; +// Watchman reports the stats of regular files, which the node crawler leaves +// to be filled in when a file is first read. +const getCases = (crawlerName: string) => { + const regularFile = + crawlerName === 'watchman' + ? [expect.any(Number), 245, 0, null, 0] + : [null, null, 0, null, 0]; + return [ + [ + true, + new Map([ + ['foo.js', regularFile], + [join('directory', 'bar.js'), regularFile], + [ + 'link-to-directory', + [expect.any(Number), 9, 0, null, expect.oneOf(1, 'directory')], + ], + [ + 'link-to-foo.js', + [expect.any(Number), 6, 0, null, expect.oneOf(1, 'foo.js')], + ], + ]), + ], + [ + false, + new Map([ + [join('directory', 'bar.js'), regularFile], + ['foo.js', regularFile], + ]), + ], + ]; +}; describe.each(Object.keys(CRAWLERS))( 'Crawler integration tests (%s)', @@ -103,7 +111,7 @@ describe.each(Object.keys(CRAWLERS))( const crawl = CRAWLERS[crawlerName]; const maybeTest = crawl ? test : test.skip; - maybeTest.each(CASES)( + maybeTest.each(getCases(crawlerName))( 'Finds the expected files (includeSymlinks: %s)', async (includeSymlinks, expectedChangedFiles) => { invariant(crawl, 'crawl should not be null within maybeTest'); diff --git a/packages/metro-file-map/src/crawlers/__tests__/node-test.js b/packages/metro-file-map/src/crawlers/__tests__/node-test.js index e8e247c9d5..360c478ec7 100644 --- a/packages/metro-file-map/src/crawlers/__tests__/node-test.js +++ b/packages/metro-file-map/src/crawlers/__tests__/node-test.js @@ -276,16 +276,134 @@ describe.each([['win32'], ['posix']])('node crawler on %s', platform => { roots: [p('/project/fruits')], }); + // New files are recorded without a stat. expect(changedFiles).toEqual( createMap({ - 'fruits/directory/strawberry.js': [33, 42, 0, null, 0], - 'fruits/tomato.js': [32, 42, 0, null, 0], + 'fruits/directory/strawberry.js': [null, null, 0, null, 0], + 'fruits/tomato.js': [null, null, 0, null, 0], }), ); expect(removedFiles).toEqual(new Set()); // once for /project/fruits, once for /project/fruits/directory expect(fs.readdir).toHaveBeenCalledTimes(2); - // once for strawberry.js, once for tomato.js + expect(fs.lstat).not.toHaveBeenCalled(); + }); + + test("does not stat a file that was not stat'd before", async () => { + nodeCrawl = require('../node').default; + const fs = require('node:fs'); + + // Neither file has any data derived from its content, so neither can be + // stale, whatever has changed on disk. + const files = createMap({ + 'fruits/directory/strawberry.js': [null, null, 0, null, 0], + 'fruits/tomato.js': [null, null, 0, null, 0], + }); + + const {changedFiles, removedFiles} = await nodeCrawl({ + console: global.console, + previousState: {fileSystem: getFS(files)}, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: [p('/project/fruits')], + }); + + expect(changedFiles).toEqual(new Map()); + expect(removedFiles).toEqual(new Set()); + expect(fs.lstat).not.toHaveBeenCalled(); + }); + + test("stats only the files that were stat'd before, without looking them up", async () => { + nodeCrawl = require('../node').default; + const fs = require('node:fs'); + + const files = createMap({ + 'fruits/directory/strawberry.js': [null, null, 0, null, 0], + 'fruits/tomato.js': [30, 40, 1, 'abc', 0], + }); + const fileSystem = getFS(files); + // The comparison with the previous file map finds them. + const lookup = jest.spyOn(fileSystem, 'lookup'); + const linkStats = jest.spyOn(fileSystem, 'linkStats'); + + const {changedFiles} = await nodeCrawl({ + console: global.console, + previousState: {fileSystem}, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: [p('/project/fruits')], + }); + + // tomato.js's mtime is now 32, so its SHA-1 is no longer valid. + expect(changedFiles).toEqual( + createMap({'fruits/tomato.js': [32, 42, 0, null, 0]}), + ); + expect(fs.lstat).toHaveBeenCalledTimes(1); + expect(fs.lstat).toHaveBeenCalledWith( + p('/project/fruits/tomato.js'), + expect.any(Function), + ); + expect(lookup).not.toHaveBeenCalled(); + expect(linkStats).not.toHaveBeenCalled(); + }); + + test('does not compare a file with one reached through a previous symlink', async () => { + nodeCrawl = require('../node').default; + + // Before, `directory` was a symlink to `other`, whose strawberry.js has the + // same modified time as the crawled one (as a hard link would). + const files = createMap({ + 'fruits/directory': [0, 0, 0, null, 'fruits/other'], + 'fruits/other/strawberry.js': [32, 42, 1, 'abc', 0], + 'fruits/tomato.js': [null, null, 0, null, 0], + }); + + const {changedFiles, removedFiles} = await nodeCrawl({ + console: global.console, + previousState: {fileSystem: getFS(files)}, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: [p('/project/fruits')], + }); + + expect(changedFiles).toEqual( + createMap({'fruits/directory/strawberry.js': [null, null, 0, null, 0]}), + ); + expect(removedFiles).toEqual( + new Set([p('fruits/directory'), p('fruits/other/strawberry.js')]), + ); + }); + + test("a scoped recrawl stats files that weren't stat'd before", async () => { + nodeCrawl = require('../node').default; + const fs = require('node:fs'); + + const files = createMap({ + 'fruits/directory/strawberry.js': [null, null, 0, null, 0], + 'fruits/tomato.js': [null, null, 0, null, 0], + }); + + const {changedFiles} = await nodeCrawl({ + console: global.console, + previousState: {fileSystem: getFS(files)}, + extensions: ['js'], + ignore: pearMatcher, + rootDir, + roots: [p('/project/fruits')], + subpath: p('fruits'), + }); + + // Their contents may have been read outside the file map, so they're + // reported as changed. + expect(changedFiles).toEqual( + createMap({ + 'fruits/directory/strawberry.js': [expect.any(Number), 42, 0, null, 0], + 'fruits/tomato.js': [expect.any(Number), 42, 0, null, 0], + }), + ); expect(fs.lstat).toHaveBeenCalledTimes(2); }); @@ -413,8 +531,9 @@ describe.each([['win32'], ['posix']])('node crawler on %s', platform => { ['fruits/directory/strawberry.js', 'fruits/tomato.js'].map(p), ); expect(ignore).toHaveBeenCalledWith(p('/project')); - expect(fs.lstat).toHaveBeenCalledWith( - p('/project/fruits/tomato.js'), + expect(fs.readdir).toHaveBeenCalledWith( + p('/project/fruits/directory'), + expect.anything(), expect.any(Function), ); }); diff --git a/packages/metro-file-map/src/crawlers/node/index.js b/packages/metro-file-map/src/crawlers/node/index.js index 2808eea959..642d76399d 100644 --- a/packages/metro-file-map/src/crawlers/node/index.js +++ b/packages/metro-file-map/src/crawlers/node/index.js @@ -11,13 +11,25 @@ import type {CrawlerOptions, CrawlResult, FileData} from '../../flow-types'; +import H from '../../constants'; import {RootPathUtils} from '../../lib/RootPathUtils'; import fs from 'node:fs'; import * as path from 'node:path'; function find(options: CrawlerOptions): Promise { - const {console, extensions, ignore, includeSymlinks, rootDir, roots} = - options; + const { + console, + extensions, + ignore, + includeSymlinks, + rootDir, + roots, + subpath, + } = options; + // A recrawl while watching diffs against the live file system, whose + // consumers (e.g. a package.json cache) may hold contents read without a + // stat, so it stats every file. + const statRegularFiles = subpath != null; const result: FileData = new Map(); const pathUtils = new RootPathUtils(rootDir); const exts = new Set(extensions); @@ -80,6 +92,14 @@ function find(options: CrawlerOptions): Promise { continue; } + if (!isSymbolicLink && !statRegularFiles) { + // Not stat'd (see `FileMetadata`). Those that were stat'd before + // are stat'd once the difference is known, by + // `dropUnmodifiedFiles`. + result.set(childNormal, [null, null, 0, null, 0]); + continue; + } + activeCalls++; fs.lstat(file, (err, stat) => { activeCalls--; @@ -132,10 +152,54 @@ export default async function nodeCrawl( abortSignal?.throwIfAborted(); - const difference = previousState.fileSystem.getDifference(fileData, { - subpath, - }); + const {changedFiles, removedFiles, filesToStat} = + previousState.fileSystem.getDifference(fileData, {subpath}); + await dropUnmodifiedFiles(options.rootDir, changedFiles, filesToStat); perfLogger?.point('nodeCrawl_end'); - return difference; + return {changedFiles, removedFiles}; +} + +/** + * Changed files crawled without a stat that were stat'd before may have data + * derived from what they held then. Stat those, drop any whose mtime matches, + * and record the stats of the rest. + */ +async function dropUnmodifiedFiles( + rootDir: string, + changedFiles: FileData, + // Each with the mtime it had before + filesToStat: ReadonlyMap, +): Promise { + // None if the previous file map is empty, as on a cold start + if (filesToStat.size === 0) { + return; + } + const pathUtils = new RootPathUtils(rootDir); + const stats: Array> = []; + for (const [normalPath, metadata] of changedFiles) { + const modifiedTime = filesToStat.get(normalPath); + if (modifiedTime == null) { + continue; + } + stats.push( + new Promise(resolve => { + fs.lstat(pathUtils.normalToAbsolute(normalPath), (err, stat) => { + if (!err && stat) { + const mtime = stat.mtime.getTime(); + if (mtime === modifiedTime) { + changedFiles.delete(normalPath); + } else { + metadata[H.MTIME] = mtime; + metadata[H.SIZE] = stat.size; + } + } + // Otherwise deleted since it was crawled, which processing will + // discover. + resolve(); + }); + }), + ); + } + await Promise.all(stats); } diff --git a/packages/metro-file-map/src/flow-types.js b/packages/metro-file-map/src/flow-types.js index 633a328bf3..988326af0d 100644 --- a/packages/metro-file-map/src/flow-types.js +++ b/packages/metro-file-map/src/flow-types.js @@ -343,8 +343,13 @@ export type IgnoreMatcher = (item: string) => boolean; export type FileData = Map; export type FileMetadata = [ + // `mtime` and `size` are null for a regular file that hasn't been stat'd, + // as the node crawler lists regular files without a stat. Such a file has + // no derived data: its mtime and size are recorded, from a stat taken before + // it's read, when a SHA-1 or plugin data is first derived from it. So a file + // without an mtime has nothing that could be stale. /* mtime */ ?number, - /* size */ number, + /* size */ ?number, /* visited */ 0 | 1, /* sha1 */ ?string, // A string is the symlink's target, lexically resolved to a normal path with @@ -368,6 +373,9 @@ export interface FileSystem { * Given a map of files, determine which of them are new or modified * (changedFiles), and which of them are missing from the input * (removedFiles), vs the current state of this instance of FileSystem. + * Regular files given without a modified time, that were regular files with + * one here, may be unmodified. They are included in changedFiles, and also + * in filesToStat with the modified time they had here. */ getDifference( files: FileData, @@ -382,6 +390,7 @@ export interface FileSystem { ): { changedFiles: FileData, removedFiles: Set, + filesToStat: Map, }; getSerializableSnapshot(): CacheData['fileSystemData']; getSha1(file: Path): ?string; @@ -425,6 +434,9 @@ export interface FileSystem { /** * Analogous to posix lstat. If the file at `file` is a symlink, return * information about the symlink without following it. + * + * `modifiedTime` and `size` are null for a regular file that hasn't been + * stat'd (see `FileMetadata`). */ linkStats(file: Path): ?FileStats; @@ -680,11 +692,15 @@ export type WorkerMessage = Readonly<{ filePath: string, maybeReturnContent: boolean, pluginsToRun: ReadonlyArray, + // Whether to stat the file before reading it, and return the result as + // `stats`. Set for a file that hasn't been stat'd (see `FileMetadata`). + returnStats: boolean, }>; export type WorkerMetadata = Readonly<{ sha1?: ?string, content?: ?Buffer, + stats?: ?Readonly<{mtime: number, size: number}>, pluginData?: ReadonlyArray, }>; diff --git a/packages/metro-file-map/src/index.js b/packages/metro-file-map/src/index.js index 7f3df73144..a62ae0e7e4 100644 --- a/packages/metro-file-map/src/index.js +++ b/packages/metro-file-map/src/index.js @@ -172,7 +172,7 @@ export type { // This should be bumped whenever a code change to `metro-file-map` itself // would cause a change to the cache data structure and/or content (for a given // filesystem state and build parameters). -const CACHE_BREAKER = '14'; +const CACHE_BREAKER = '15'; const CHANGE_INTERVAL = 30; diff --git a/packages/metro-file-map/src/lib/FileProcessor.js b/packages/metro-file-map/src/lib/FileProcessor.js index fcec8441c9..72bc41346f 100644 --- a/packages/metro-file-map/src/lib/FileProcessor.js +++ b/packages/metro-file-map/src/lib/FileProcessor.js @@ -170,7 +170,9 @@ export class FileProcessor { filePath: absolutePath, maybeReturnContent: false, pluginsToRun: [pluginIdx], + returnStats: fileMetadata[H.MTIME] == null, }); + applyStats(reply, fileMetadata); // `undefined` is reserved to mean that no worker has run. const pluginData = reply.pluginData?.[0] ?? null; // $FlowFixMe[invalid-tuple-index] @@ -219,6 +221,7 @@ export class FileProcessor { filePath: this.#rootPathUtils.normalToAbsolute(normalPath), maybeReturnContent, pluginsToRun, + returnStats: fileMetadata[H.MTIME] == null, }; } @@ -270,6 +273,7 @@ function processWorkerReply( fileMetadata: FileMetadata, ) { fileMetadata[H.VISITED] = 1; + applyStats(metadata, fileMetadata); const pluginData = metadata.pluginData; if (pluginData) { for (const [i, pluginIdx] of pluginsRun.entries()) { @@ -286,6 +290,16 @@ function processWorkerReply( return metadata.content; } +// Records the stats of a file that hadn't been stat'd, taken as data was first +// derived from it (see `FileMetadata`). +function applyStats(metadata: WorkerMetadata, fileMetadata: FileMetadata) { + const {stats} = metadata; + if (stats != null) { + fileMetadata[H.MTIME] = stats.mtime; + fileMetadata[H.SIZE] = stats.size; + } +} + function normalizeWorkerError(mixedError: ?Error | string): MaybeCodedError { if ( mixedError == null || diff --git a/packages/metro-file-map/src/lib/TreeFS.js b/packages/metro-file-map/src/lib/TreeFS.js index 8583e45403..a2b1ea7ec2 100644 --- a/packages/metro-file-map/src/lib/TreeFS.js +++ b/packages/metro-file-map/src/lib/TreeFS.js @@ -306,9 +306,11 @@ export default class TreeFS implements MutableFileSystem { ): { changedFiles: FileData, removedFiles: Set, + filesToStat: Map, } { const changedFiles: FileData = new Map(files); const removedFiles: Set = new Set(); + const filesToStat: Map = new Map(); const subpath = options?.subpath; // If a subpath is specified, start iteration from that node @@ -324,13 +326,13 @@ export default class TreeFS implements MutableFileSystem { ); if (!lookupResult.exists) { // Directory doesn't exist, nothing to compare - all files are new - return {changedFiles, removedFiles}; + return {changedFiles, removedFiles, filesToStat}; } if (!isDirectory(lookupResult.node)) { // A file or symlink has been replaced by this directory, so it is // removed and everything under the directory is new. removedFiles.add(lookupResult.canonicalPath); - return {changedFiles, removedFiles}; + return {changedFiles, removedFiles, filesToStat}; } rootNode = lookupResult.node; prefix = lookupResult.canonicalPath; @@ -358,6 +360,14 @@ export default class TreeFS implements MutableFileSystem { ) { // Types and modified time match - not changed. changedFiles.delete(canonicalPath); + } else if ( + isRegularFile(metadata) && + newMetadata[H.MTIME] == null && + metadata[H.MTIME] == null + ) { + // A regular file not stat'd before or now, so nothing could be + // stale (see `FileMetadata`). + changedFiles.delete(canonicalPath); } else if ( newMetadata[H.SHA1] != null && newMetadata[H.SHA1] === metadata[H.SHA1] && @@ -367,6 +377,13 @@ export default class TreeFS implements MutableFileSystem { const updatedMetadata = [...metadata]; updatedMetadata[H.MTIME] = newMetadata[H.MTIME]; changedFiles.set(canonicalPath, updatedMetadata); + } else if ( + isRegularFile(metadata) && + newMetadata[H.MTIME] == null && + metadata[H.MTIME] != null + ) { + // Stat'd before but not now, so it may be unmodified. + filesToStat.set(canonicalPath, metadata[H.MTIME]); } } else { removedFiles.add(canonicalPath); @@ -375,6 +392,7 @@ export default class TreeFS implements MutableFileSystem { return { changedFiles, removedFiles, + filesToStat, }; } diff --git a/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js b/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js index f1c9582717..2dfbd426d3 100644 --- a/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js +++ b/packages/metro-file-map/src/lib/__tests__/FileProcessor-test.js @@ -306,6 +306,7 @@ describe('processBatch', () => { filePath: p('/root/node_modules/pkg/index.js'), maybeReturnContent: false, pluginsToRun: [0], + returnStats: false, }); }, ); @@ -364,6 +365,7 @@ describe('processBatch', () => { filePath: p('/root/node_modules/pkg/package.json'), maybeReturnContent: false, pluginsToRun: [], + returnStats: false, }); }); }); @@ -399,6 +401,7 @@ describe('processBatch', () => { filePath: absolutePath, maybeReturnContent: false, pluginsToRun: [1], + returnStats: false, }); expect(MockJestWorker).not.toHaveBeenCalled(); // Only this plugin's slot is written. In particular the file is not @@ -550,6 +553,7 @@ describe('processBatch', () => { describe('processRegularFile', () => { let FileProcessor; const mockReadFileSync = jest.fn(); + const mockStatSync = jest.fn(); beforeEach(() => { jest.resetModules(); @@ -557,6 +561,7 @@ describe('processRegularFile', () => { jest.unmock('../../worker.js'); jest.mock('node:fs', () => ({ readFileSync: mockReadFileSync, + statSync: mockStatSync, })); FileProcessor = require('../FileProcessor').FileProcessor; }); @@ -584,6 +589,29 @@ describe('processRegularFile', () => { }); expect(metadata[H.SHA1]).toMatch(/^[a-f0-9]{40}$/); + // A known modified time is kept + expect(mockStatSync).not.toHaveBeenCalled(); + expect(metadata[H.MTIME]).toBe(123); + }); + + test('records the stats of a file crawled without them', () => { + const processor = new FileProcessor(defaultOptions); + const metadata: FileMetadata = [null, null, 0, null, 0, null]; + + mockReadFileSync.mockReturnValue(Buffer.from('hello world')); + mockStatSync.mockReturnValue({mtime: new Date(1700000000000), size: 11}); + + processor.processRegularFile('file.js', metadata, { + computeSha1: true, + maybeReturnContent: false, + }); + + expect(mockStatSync).toHaveBeenCalledWith( + path.resolve(defaultOptions.rootDir, 'file.js'), + ); + expect(metadata[H.SHA1]).toMatch(/^[a-f0-9]{40}$/); + expect(metadata[H.MTIME]).toBe(1700000000000); + expect(metadata[H.SIZE]).toBe(11); }); }); diff --git a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js index 7c69fdd5cd..74baaa404d 100644 --- a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js +++ b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js @@ -420,6 +420,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { p('abs-link-out'), p('root'), ]), + filesToStat: new Map(), }); }); @@ -441,6 +442,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { p('foo/link-to-bar.js'), p('foo/link-to-another.js'), ]), + filesToStat: new Map(), }); }); @@ -484,6 +486,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { [p('nonexistent/file.js'), [123, 0, 0, null, 0, null]], ]), removedFiles: new Set(), + filesToStat: new Map(), }); }); @@ -500,10 +503,43 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { expect(tfs.getDifference(newFiles, {subpath})).toEqual({ changedFiles: newFiles, removedFiles: new Set([subpath]), + filesToStat: new Map(), }); }, ); + test('returns regular files given without an mtime that had one, with that mtime', () => { + const newFiles: FileData = new Map([ + // Stat'd before, not now + [p('foo/another.js'), [null, null, 0, null, 0, null]], + // Stat'd before with an mtime of 0, outside rootDir, not now + [p('../outside/external.js'), [null, null, 0, null, 0, null]], + // Stat'd before and now, with the same mtime + [p('bar.js'), [234, 3, 0, null, 0, null]], + // A symlink before, not stat'd now + [p('link-to-self'), [null, null, 0, null, 0, null]], + // New, not stat'd + [p('new-file.js'), [null, null, 0, null, 0, null]], + ]); + + const {changedFiles, filesToStat} = tfs.getDifference(newFiles); + + expect(filesToStat).toEqual( + new Map([ + [p('foo/another.js'), 123], + [p('../outside/external.js'), 0], + ]), + ); + expect([...changedFiles.keys()].sort()).toEqual( + [ + p('foo/another.js'), + p('../outside/external.js'), + p('link-to-self'), + p('new-file.js'), + ].sort(), + ); + }); + test('with empty subpath only considers files under the root', () => { const newFiles: FileData = new Map([ [p('foo/another.js'), [123, 2, 0, null, 0, 'another']], diff --git a/packages/metro-file-map/src/worker.js b/packages/metro-file-map/src/worker.js index 3d6110a577..35f568b7fd 100644 --- a/packages/metro-file-map/src/worker.js +++ b/packages/metro-file-map/src/worker.js @@ -47,8 +47,17 @@ class Worker { processFile(data /*: WorkerMessage */) /*: WorkerMetadata */ { let content; let sha1; + let stats; - const {computeSha1, filePath, pluginsToRun} = data; + const {computeSha1, filePath, pluginsToRun, returnStats} = data; + + // Stat before deriving anything, so that the recorded mtime is never newer + // than the content the data was derived from. `getTime()` truncates to + // whole milliseconds, as the crawler's mtimes are. + if (returnStats) { + const {mtime, size} = fs.statSync(filePath); + stats = {mtime: mtime.getTime(), size}; + } const getContent = () /*: Buffer */ => { if (content == null) { @@ -69,8 +78,8 @@ class Worker { } return content && data.maybeReturnContent - ? {content, pluginData, sha1} - : {pluginData, sha1}; + ? {content, pluginData, sha1, stats} + : {pluginData, sha1, stats}; } }