vfs: integrate with CJS and ESM module loaders

benchmark/vfs/bench-fs-dispatch.js

@@ -0,0 +1,73 @@
+'use strict';
+
+// Measures the dispatch overhead the VFS layer adds to fs operations
+// against real-filesystem paths (paths that no VFS claims). The hot
+// path in lib/fs.js is:
+//
+//   const h = vfsState.handlers;
+//   if (h !== null) { const r = h.opSync(path, ...); if (r !== undefined) return r; }
+//   binding.op(getValidatedPath(path));
+//
+// With layers=0 the VFS module is never required and `h === null` is
+// the first thing fs sees. With layers>=1 the handler normalizes the
+// path once and rejects it with a single prefix comparison against the
+// reserved VFS namespace - the number of mounted layers should not
+// matter. The benchmark mounts N VFSes and probes a real file under
+// __dirname, so every call falls through after the namespace check
+// declines the path.
+
+const common = require('../common.js');
+const fs = require('fs');
+const path = require('path');
+
+const bench = common.createBenchmark(main, {
+  n: [3e5],
+  op: ['statSync', 'existsSync', 'accessSync', 'readFileSync'],
+  // 0 = VFS module never loaded (true baseline)
+  // >=1 = that many VFS instances mounted
+  layers: [0, 1, 2, 5, 10],
+}, {
+  flags: ['--experimental-vfs', '--no-warnings'],
+});
+
+function mountLayers(count) {
+  const vfs = require('node:vfs');
+  const handles = [];
+  for (let i = 0; i < count; i++) {
+    const v = vfs.create();
+    v.mount();
+    handles.push(v);
+  }
+  return handles;
+}
+
+function main({ n, op, layers }) {
+  const handles = layers > 0 ? mountLayers(layers) : null;
+
+  const target = layers === 0 ? __filename : path.join(__dirname, path.basename(__filename));
+
+  // Warm-up - get the JIT past the first-call icache + IC misses so we
+  // measure steady-state dispatch cost, not first-call resolution.
+  for (let i = 0; i < 1000; i++) {
+    if (op === 'statSync') fs.statSync(target);
+    else if (op === 'existsSync') fs.existsSync(target);
+    else if (op === 'accessSync') fs.accessSync(target);
+    else fs.readFileSync(target);
+  }
+
+  bench.start();
+  if (op === 'statSync') {
+    for (let i = 0; i < n; i++) fs.statSync(target);
+  } else if (op === 'existsSync') {
+    for (let i = 0; i < n; i++) fs.existsSync(target);
+  } else if (op === 'accessSync') {
+    for (let i = 0; i < n; i++) fs.accessSync(target);
+  } else {
+    for (let i = 0; i < n; i++) fs.readFileSync(target);
+  }
+  bench.end(n);
+
+  if (handles) {
+    for (const v of handles) v.unmount();
+  }
+}

doc/api/vfs.md

@@ -61,6 +61,11 @@ callback-based, and promise-based file system methods that mirror the
 shape of the [`node:fs`][] API. All paths are POSIX-style and absolute
 (starting with `/`).
 
+By default, the file tree is private to the VFS instance. To expose
+it through the global `node:fs` module, `require()`, and `import`,
+call [`vfs.mount()`][]; call [`vfs.unmount()`][] (or rely on a
+`using` declaration) to detach again.
+
 ## `vfs.create([provider][, options])`
 
 <!-- YAML
@@ -107,6 +112,126 @@ added: v26.4.0
   * `emitExperimentalWarning` {boolean} Whether to emit the experimental
     warning. **Default:** `true`.
 
+### `vfs.mount()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Returns: {string} The absolute mount point.
+
+Mounts the virtual file system and returns the resulting mount point.
+After mounting, files in the VFS can be accessed through the
+`node:fs` module and resolved through `require()` and `import`
+using paths under the returned mount point.
+
+Mount points always live inside a reserved namespace,
+`${os.devNull}/vfs/<layerId>`. Because [`os.devNull`][] is a
+character device (POSIX) or a device-namespace path (Windows) that
+cannot have child file system entries, no real file-system path can
+exist under this namespace: virtual paths never conflate with (or
+shadow) real paths, and the layer that owns a path is visible in the
+path itself.
+
+```cjs
+const vfs = require('node:vfs');
+const fs = require('node:fs');
+
+const myVfs = vfs.create();
+myVfs.writeFileSync('/data.txt', 'Hello');
+const mountPoint = myVfs.mount();
+// e.g. '/dev/null/vfs/0'
+
+fs.readFileSync(`${mountPoint}/data.txt`, 'utf8'); // 'Hello'
+```
+
+Each `VirtualFileSystem` instance may be mounted at most once at a
+time. Attempting to mount an already-mounted instance throws
+`ERR_INVALID_STATE`. Because each instance mounts inside its own
+per-layer namespace, mounts from different instances can never
+overlap.
+
+The VFS supports the [Explicit Resource Management][] proposal. Use
+a `using` declaration to unmount automatically when leaving scope:
+
+```cjs
+const vfs = require('node:vfs');
+const fs = require('node:fs');
+
+let mountPoint;
+{
+  using myVfs = vfs.create();
+  myVfs.writeFileSync('/data.txt', 'Hello');
+  mountPoint = myVfs.mount();
+
+  fs.readFileSync(`${mountPoint}/data.txt`, 'utf8'); // 'Hello'
+} // VFS is automatically unmounted here
+
+fs.existsSync(`${mountPoint}/data.txt`); // false
+```
+
+### `vfs.unmount()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Unmounts the virtual file system. After unmounting, virtual files
+are no longer reachable through `node:fs`, `require()`, or `import`.
+The same instance may be mounted again by calling `mount()`.
+
+This method is idempotent: calling `unmount()` on a VFS that is not
+currently mounted has no effect.
+
+### `vfs.mounted`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {boolean}
+
+`true` while the VFS is mounted; `false` otherwise.
+
+### `vfs.mountPoint`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {string | null}
+
+The current mount point as an absolute string (the value returned by
+the last [`vfs.mount()`][] call), or `null` when the VFS is not
+mounted.
+
+### `vfs.layerId`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {number}
+
+A per-process monotonically increasing identifier assigned at
+construction. The id is stable across `mount()` / `unmount()` cycles
+for the lifetime of the instance, and is independent of the order in
+which VFS layers are mounted.
+
+The layer id forms the `<id>` segment of the reserved mount
+namespace, so every path served by this instance carries the id, and
+it appears in the `NODE_DEBUG=vfs` output for `register` and
+`deregister` events.
+
+```cjs
+const vfs = require('node:vfs');
+
+const a = vfs.create();
+const b = vfs.create();
+console.log(a.layerId); // e.g. 0
+console.log(b.layerId); // a.layerId + 1
+```
+
 ### `vfs.provider`
 
 <!-- YAML
@@ -195,6 +320,71 @@ The promise namespace mirrors `fs.promises` and includes `readFile`,
 `access`, `rm`, `truncate`, `link`, `mkdtemp`, `chmod`, `chown`, `lchown`,
 `utimes`, `lutimes`, `open`, `lchmod`, and `watch`.
 
+## Module loader integration
+
+Once a `VirtualFileSystem` is mounted, paths under the mount point
+participate in module resolution and loading. Both
+`require()` / `require.resolve()` (CommonJS) and `import` /
+`import.meta.resolve()` (ECMAScript modules) consult the VFS through
+the same toggleable hooks that `node:fs` uses, so files served from
+the VFS are first-class modules: `package.json` is honoured,
+extensionless files are sniffed for Wasm vs. JavaScript, conditional
+`exports` / `imports` work, and so on.
+
+```cjs
+const vfs = require('node:vfs');
+
+const myVfs = vfs.create();
+myVfs.mkdirSync('/lib');
+myVfs.writeFileSync('/lib/greet.js', 'module.exports = () => "hi";');
+myVfs.writeFileSync(
+  '/lib/package.json', '{"main": "./greet.js"}');
+const mountPoint = myVfs.mount();
+
+const greet = require(`${mountPoint}/lib`);
+console.log(greet()); // 'hi'
+
+myVfs.unmount();
+```
+
+For ECMAScript modules, convert mounted paths to `file:` URLs before
+passing them to dynamic `import()`. This keeps VFS imports portable on
+Windows, where mounted paths use Windows path syntax.
+
+```mjs
+import { pathToFileURL } from 'node:url';
+import vfs from 'node:vfs';
+
+const myVfs = vfs.create();
+myVfs.writeFileSync('/mod.mjs', 'export const value = 42;');
+const mountPoint = myVfs.mount();
+
+const { value } = await import(
+  pathToFileURL(`${mountPoint}/mod.mjs`).href,
+);
+console.log(value); // 42
+
+myVfs.unmount();
+```
+
+Module identity follows the path: `__filename` and `module.filename`
+are the plain absolute path of the module under the mount point, and
+`import.meta.url` is the corresponding `file:` URL, with no synthetic
+decorations. Importing the same virtual path repeatedly, including
+through `import.meta.resolve()`, yields the same module instance,
+exactly as for real files.
+
+Calling [`vfs.unmount()`][] invalidates the modules that were loaded
+from the mount point: a subsequent `require()` or `import` of a path
+under a re-created mount re-reads the file from the newly mounted
+VFS rather than returning a stale module. Modules loaded from other
+VFS instances or from the real file system are unaffected.
+
+Mounting and unmounting do not invalidate ESM modules that are
+already executing. As with any other module-system teardown,
+unmounting a VFS while the import graph below it is still loading is
+the caller's responsibility to avoid.
+
 ## Class: `VirtualProvider`
 
 <!-- YAML
@@ -318,10 +508,14 @@ fields use synthetic but stable values:
 * `blocks` is `Math.ceil(size / 512)`.
 * Times default to the moment the entry was created/last modified.
 
+[Explicit Resource Management]: https://github.com/tc39/proposal-explicit-resource-management
 [`MemoryProvider`]: #class-memoryprovider
 [`RealFSProvider`]: #class-realfsprovider
 [`VirtualFileSystem`]: #class-virtualfilesystem
 [`VirtualProvider`]: #class-virtualprovider
 [`fs.BigIntStats`]: fs.md#class-fsbigintstats
 [`fs.Stats`]: fs.md#class-fsstats
 [`node:fs`]: fs.md
+[`os.devNull`]: os.md#osdevnull
+[`vfs.mount()`]: #vfsmount
+[`vfs.unmount()`]: #vfsunmount

lib/fs.js

@@ -2044,8 +2044,13 @@ function fstatSync(fd, options = { __proto__: null, bigint: false }) {
 function lstatSync(path, options = { __proto__: null, bigint: false, throwIfNoEntry: true }) {
   const h = vfsState.handlers;
   if (h !== null) {
-    const result = h.lstatSync(path, options);
-    if (result !== undefined) return result;
+    try {
+      const result = h.lstatSync(path, options);
+      if (result !== undefined) return result;
+    } catch (err) {
+      if (err?.code === 'ENOENT' && options?.throwIfNoEntry === false) return;
+      throw err;
+    }
   }
   path = getValidatedPath(path);
   if (permission.isEnabled() && !permission.has('fs.read', path)) {
@@ -2078,8 +2083,13 @@ function lstatSync(path, options = { __proto__: null, bigint: false, throwIfNoEn
 function statSync(path, options = { __proto__: null, bigint: false, throwIfNoEntry: true }) {
   const h = vfsState.handlers;
   if (h !== null) {
-    const result = h.statSync(path, options);
-    if (result !== undefined) return result;
+    try {
+      const result = h.statSync(path, options);
+      if (result !== undefined) return result;
+    } catch (err) {
+      if (err?.code === 'ENOENT' && options?.throwIfNoEntry === false) return undefined;
+      throw err;
+    }
   }
   const stats = binding.stat(
     getValidatedPath(path),

lib/internal/modules/cjs/loader.js

@@ -113,6 +113,7 @@ const kFormat = Symbol('kFormat');
 
 // Set first due to cycle with ESM loader functions.
 module.exports = {
+  purgeModuleCachesForPrefix,
   kModuleSource,
   kModuleExport,
   kModuleExportNames,
@@ -155,14 +156,15 @@ const {
 } = internalBinding('contextify');
 
 const assert = require('internal/assert');
-const fs = require('fs');
 const path = require('path');
-const internalFsBinding = internalBinding('fs');
 const { safeGetenv } = internalBinding('credentials');
 const {
+  cleanForVfsPrefix,
   getCjsConditions,
   getCjsConditionsArray,
   initializeCjsConditions,
+  loaderReadFile,
+  loaderStat,
   loadBuiltinModule,
   makeRequireFunction,
   setHasStartedUserCJSExecution,
@@ -278,14 +280,31 @@ function stat(filename) {
     const result = statCache.get(filename);
     if (result !== undefined) { return result; }
   }
-  const result = internalFsBinding.internalModuleStat(filename);
+  const result = loaderStat(filename);
   if (statCache !== null && result >= 0) {
     // Only set cache when `internalModuleStat(filename)` succeeds.
     statCache.set(filename, result);
   }
   return result;
 }
 
+/**
+ * Drop every module-loader cache entry (Module._cache,
+ * Module._pathCache, the stat cache) whose filename lives under
+ * `mountPoint`. Called by internal/vfs/setup.js when a VFS is
+ * unmounted. VFS mount points live in a reserved namespace that
+ * cannot collide with real paths, so a prefix scan is exact:
+ * real-fs entries and other-VFS entries never match.
+ * @param {string} mountPoint Absolute mount-point path
+ */
+function purgeModuleCachesForPrefix(mountPoint) {
+  cleanForVfsPrefix(Module._cache, mountPoint);
+  // Module._pathCache keys are `request\0parent` tokens - the resolved
+  // filename lives in the value, so predicate on the value.
+  cleanForVfsPrefix(Module._pathCache, mountPoint, true);
+  if (statCache !== null) { cleanForVfsPrefix(statCache, mountPoint); }
+}
+
 let _stat = stat;
 ObjectDefineProperty(Module, '_stat', {
   __proto__: null,
@@ -1248,7 +1267,7 @@ function defaultLoadImpl(filename, format) {
     case 'module-typescript':
     case 'commonjs-typescript':
     case 'typescript': {
-      return fs.readFileSync(filename, 'utf8');
+      return loaderReadFile(filename, 'utf8');
     }
     case 'builtin':
       return null;