metro-file-map: Lazily stat files in the node crawler

packages/metro-file-map/API.md

@@ -176,7 +176,11 @@ interface FileSystem_2 {
     options?: Readonly<{
       subpath?: string | undefined;
     }>,
-  ): {changedFiles: FileData; removedFiles: Set<string>};
+  ): {
+    changedFiles: FileData;
+    removedFiles: Set<string>;
+    filesToStat: Map<string, number>;
+  };
   getOrComputeSha1(file: Path): Promise<null | undefined | {sha1: string; content?: Buffer | undefined}>;
   getSerializableSnapshot(): CacheData['fileSystemData'];
   getSha1(file: Path): null | undefined | string;

packages/metro-file-map/src/__tests__/addRoot-test.js

@@ -13,6 +13,7 @@ import type {
   CacheData,
   ChangeEvent,
   CrawlerOptions,
+  FileData,
   FileMetadata,
   WatcherBackendOptions,
 } from '../flow-types';
@@ -29,6 +30,16 @@ let mockFiles: Map<string, number>;
 // Absolute path -> normal (rootDir-relative, POSIX) symlink target.
 let mockSymlinks: Map<string, string>;
 
+// What the node crawler returns of `getDifference`'s result.
+const toCrawlResult = ({
+  changedFiles,
+  removedFiles,
+}: {
+  changedFiles: FileData,
+  removedFiles: Set<string>,
+  ...
+}) => ({changedFiles, removedFiles});
+
 const mockNodeCrawler = jest.fn((options: CrawlerOptions) => {
   const path = require('node:path');
   const files = new Map<string, FileMetadata>();
@@ -69,9 +80,11 @@ const mockNodeCrawler = jest.fn((options: CrawlerOptions) => {
     }
   }
   return Promise.resolve(
-    options.previousState.fileSystem.getDifference(files, {
-      subpath: options.subpath,
-    }),
+    toCrawlResult(
+      options.previousState.fileSystem.getDifference(files, {
+        subpath: options.subpath,
+      }),
+    ),
   );
 });
 
@@ -316,14 +329,16 @@ describe('FileMap.addRoot', () => {
           new Promise(resolve => {
             finishCrawl = () =>
               resolve(
-                options.previousState.fileSystem.getDifference(
-                  new Map([
-                    [
-                      path.relative(ROOT_DIR, EXTERNAL_FILE),
-                      [1, 42, 0, null, 0],
-                    ],
-                  ]),
-                  {subpath: options.subpath},
+                toCrawlResult(
+                  options.previousState.fileSystem.getDifference(
+                    new Map([
+                      [
+                        path.relative(ROOT_DIR, EXTERNAL_FILE),
+                        [1, 42, 0, null, 0],
+                      ],
+                    ]),
+                    {subpath: options.subpath},
+                  ),
                 ),
               );
             resolveStarted();
@@ -376,14 +391,16 @@ describe('FileMap.addRoot', () => {
     // Like Watchman, list a root's files under its real path.
     mockNodeCrawler.mockImplementationOnce(options =>
       Promise.resolve(
-        options.previousState.fileSystem.getDifference(
-          new Map([
-            [
-              path.relative(ROOT_DIR, path.join(SRC_DIR, 'index.js')),
-              [1, 42, 0, null, 0],
-            ],
-            [path.relative(ROOT_DIR, realFile), [1, 42, 0, null, 0]],
-          ]),
+        toCrawlResult(
+          options.previousState.fileSystem.getDifference(
+            new Map([
+              [
+                path.relative(ROOT_DIR, path.join(SRC_DIR, 'index.js')),
+                [1, 42, 0, null, 0],
+              ],
+              [path.relative(ROOT_DIR, realFile), [1, 42, 0, null, 0]],
+            ]),
+          ),
         ),
       ),
     );

packages/metro-file-map/src/__tests__/index-test.js

@@ -252,6 +252,7 @@ const assertFileSystemEqual = (fileSystem: FileSystem, fileData: FileData) => {
   expect(fileSystem.getDifference(fileData)).toEqual({
     changedFiles: new Map(),
     removedFiles: new Set(),
+    filesToStat: new Map(),
   });
 };
 
@@ -1503,6 +1504,7 @@ describe('FileMap', () => {
           filePath: p('/project/fruits/Banana.js'),
           maybeReturnContent: false,
           pluginsToRun: [0],
+          returnStats: false,
         },
       ],
       [
@@ -1511,6 +1513,7 @@ describe('FileMap', () => {
           filePath: p('/project/fruits/Pear.js'),
           maybeReturnContent: false,
           pluginsToRun: [0],
+          returnStats: false,
         },
       ],
       [
@@ -1519,6 +1522,7 @@ describe('FileMap', () => {
           filePath: p('/project/fruits/Strawberry.js'),
           maybeReturnContent: false,
           pluginsToRun: [0],
+          returnStats: false,
         },
       ],
       [
@@ -1527,6 +1531,7 @@ describe('FileMap', () => {
           filePath: p('/project/fruits/__mocks__/Pear.js'),
           maybeReturnContent: false,
           pluginsToRun: [0],
+          returnStats: false,
         },
       ],
       [
@@ -1535,6 +1540,7 @@ describe('FileMap', () => {
           filePath: p('/project/vegetables/Melon.js'),
           maybeReturnContent: false,
           pluginsToRun: [0],
+          returnStats: false,
         },
       ],
     ]);

packages/metro-file-map/src/__tests__/stats-between-starts-test.js

@@ -0,0 +1,236 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow strict-local
+ * @format
+ * @oncall react_native
+ */
+
+import type {FileSystem} from '../flow-types';
+
+import FileMap, {DiskCacheManager} from '../index';
+import FileDataPlugin from '../plugins/FileDataPlugin';
+import {createHash} from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+// The node crawler stats a regular file only once data is derived from its
+// content, and treats one that was never stat'd as unchanged. These tests
+// change files while no FileMap is running, as between two starts of Metro,
+// against a real file system and cache.
+
+jest.useRealTimers();
+
+type LengthPlugin = FileDataPlugin<?{length: number}>;
+
+let rootDir: string;
+let cacheDirectory: string;
+
+beforeEach(() => {
+  rootDir = fs.realpathSync(
+    fs.mkdtempSync(path.join(os.tmpdir(), 'metro-file-map-root-')),
+  );
+  cacheDirectory = fs.mkdtempSync(
+    path.join(os.tmpdir(), 'metro-file-map-cache-'),
+  );
+});
+
+afterEach(() => {
+  fs.rmSync(rootDir, {recursive: true, force: true});
+  fs.rmSync(cacheDirectory, {recursive: true, force: true});
+});
+
+const abs = (normalPath: string) => path.join(rootDir, normalPath);
+
+function write(normalPath: string, content: string) {
+  fs.mkdirSync(path.dirname(abs(normalPath)), {recursive: true});
+  fs.writeFileSync(abs(normalPath), content);
+}
+
+// Timestamps can be coarser than a test runs (about 15ms on Windows), so move
+// the modified time on, as an edit some time after the last start would.
+let offlineEdits = 0;
+function editOffline(normalPath: string, content: string) {
+  write(normalPath, content);
+  const seconds = Date.now() / 1000 + 60 + ++offlineEdits;
+  fs.utimesSync(abs(normalPath), seconds, seconds);
+}
+
+const sha1 = (content: string) =>
+  createHash('sha1').update(content).digest('hex');
+
+function createLengthPlugin(name: string, lazy: boolean): LengthPlugin {
+  return new FileDataPlugin({
+    name,
+    cacheKey: name,
+    worker: {
+      modulePath: require.resolve('./lazy_plugin_worker.js'),
+      setupArgs: {},
+    },
+    filter: ({normalPath}) => normalPath.endsWith('.eager.js'),
+    lazy,
+  });
+}
+
+async function session(
+  fn: (
+    fileSystem: FileSystem,
+    plugins: {eager: LengthPlugin, lazy: LengthPlugin},
+  ) => Promise<void> | void,
+): Promise<void> {
+  const eager = createLengthPlugin('eager', false);
+  const lazy = createLengthPlugin('lazy', true);
+  const fileMap = new FileMap({
+    cacheManagerFactory: ({buildParameters}) =>
+      new DiskCacheManager({buildParameters}, {cacheDirectory}),
+    enableSymlinks: true,
+    extensions: ['js'],
+    healthCheck: {
+      enabled: false,
+      interval: 10000,
+      timeout: 1000,
+      filePrefix: '.metro-file-map-health-check',
+    },
+    maxWorkers: 1,
+    plugins: [eager, lazy],
+    retainAllFiles: true,
+    rootDir,
+    roots: [rootDir],
+    useWatchman: false,
+    watch: false,
+  });
+  try {
+    const {fileSystem} = await fileMap.build();
+    await fn(fileSystem, {eager, lazy});
+  } finally {
+    await fileMap.end();
+  }
+}
+
+test('a SHA-1 is recomputed after an edit', async () => {
+  write('a.js', 'before');
+  await session(async fileSystem => {
+    expect(await fileSystem.getOrComputeSha1(abs('a.js'))).toMatchObject({
+      sha1: sha1('before'),
+    });
+  });
+
+  editOffline('a.js', 'after');
+
+  await session(async fileSystem => {
+    expect(await fileSystem.getOrComputeSha1(abs('a.js'))).toMatchObject({
+      sha1: sha1('after'),
+    });
+  });
+});
+
+test("an eager plugin's data is recomputed after an edit", async () => {
+  write('b.eager.js', 'before');
+  await session((_, {eager}) => {
+    expect(eager.getOrComputeFileData(abs('b.eager.js'))).toMatchObject({
+      data: {length: 6},
+    });
+  });
+
+  editOffline('b.eager.js', 'and after');
+
+  await session((_, {eager}) => {
+    expect(eager.getOrComputeFileData(abs('b.eager.js'))).toMatchObject({
+      data: {length: 9},
+    });
+  });
+});
+
+test("a lazy plugin's data is recomputed after an edit", async () => {
+  write('c.js', 'before');
+  await session((_, {lazy}) => {
+    expect(lazy.getOrComputeFileData(abs('c.js'))).toMatchObject({
+      data: {length: 6},
+    });
+  });
+
+  editOffline('c.js', 'and after');
+
+  await session((_, {lazy}) => {
+    expect(lazy.getOrComputeFileData(abs('c.js'))).toMatchObject({
+      data: {length: 9},
+    });
+  });
+});
+
+test('an edit to a file not yet read is seen when it is first read', async () => {
+  write('d.js', 'before');
+  await session(() => {});
+
+  editOffline('d.js', 'and after');
+
+  await session(async (fileSystem, {lazy}) => {
+    expect(lazy.getOrComputeFileData(abs('d.js'))).toMatchObject({
+      data: {length: 9},
+    });
+    expect(await fileSystem.getOrComputeSha1(abs('d.js'))).toMatchObject({
+      sha1: sha1('and after'),
+    });
+  });
+});
+
+test('a retargeted symlink is followed to its new target', async () => {
+  write('pkgs/a@1/index.js', 'one');
+  write('pkgs/a@2/index.js', 'two');
+  fs.mkdirSync(abs('node_modules'));
+  const link = abs('node_modules/a');
+  fs.symlinkSync(path.join('..', 'pkgs', 'a@1'), link, 'dir');
+
+  await session(async fileSystem => {
+    expect(fileSystem.lookup(abs('node_modules/a/index.js'))).toMatchObject({
+      exists: true,
+      realPath: abs('pkgs/a@1/index.js'),
+    });
+    await fileSystem.getOrComputeSha1(abs('node_modules/a/index.js'));
+  });
+
+  // As a package manager upgrading `a`
+  fs.unlinkSync(link);
+  fs.symlinkSync(path.join('..', 'pkgs', 'a@2'), link, 'dir');
+  const seconds = Date.now() / 1000 + 60;
+  // $FlowFixMe[prop-missing] - lutimesSync is missing from flow-typed's node defs
+  fs.lutimesSync(link, seconds, seconds);
+  fs.rmSync(abs('pkgs/a@1'), {recursive: true});
+
+  await session(fileSystem => {
+    expect(fileSystem.lookup(abs('node_modules/a/index.js'))).toMatchObject({
+      exists: true,
+      realPath: abs('pkgs/a@2/index.js'),
+    });
+  });
+});
+
+test('a start stats only the files with data derived from them', async () => {
+  for (let i = 0; i < 10; i++) {
+    write(`f${i}.js`, String(i));
+  }
+  await session(async fileSystem => {
+    await fileSystem.getOrComputeSha1(abs('f0.js'));
+  });
+
+  const lstat = jest.spyOn(fs, 'lstat');
+  try {
+    await session(fileSystem => {
+      // Without a stat, a file's size isn't known
+      expect(fileSystem.linkStats(abs('f1.js'))).toEqual({
+        fileType: 'f',
+        modifiedTime: null,
+        size: null,
+      });
+    });
+    expect(lstat.mock.calls.map(([filePath]) => filePath)).toEqual([
+      abs('f0.js'),
+    ]);
+  } finally {
+    lstat.mockRestore();
+  }
+});