fs: support caller-supplied readFile() buffers

Author: mcollina

doc/api/fs.md

@@ -701,6 +701,9 @@ added: v10.0.0
 * `options` {Object|string}
   * `encoding` {string|null} **Default:** `null`
   * `signal` {AbortSignal} allows aborting an in-progress readFile
+  * `buffer` {Buffer|TypedArray|DataView} A buffer to read into.
+  * `getBuffer` {Function} A synchronous function called with the file size and
+    returning the buffer to read into.
 * Returns: {Promise} Fulfills upon a successful read with the contents of the
   file. If no encoding is specified (using `options.encoding`), the data is
   returned as a {Buffer} object. Otherwise, the data will be a string.
@@ -709,6 +712,11 @@ Asynchronously reads the entire contents of a file.
 
 If `options` is a string, then it specifies the `encoding`.
 
+The `buffer` and `getBuffer` options cannot be used together. If one of them is
+provided and no encoding is specified, the returned {Buffer} is a view over the
+supplied buffer containing only the bytes read. If the supplied buffer is too
+small to contain the entire file, the operation will fail.
+
 The {FileHandle} has to support reading.
 
 If one or more `filehandle.read()` calls are made on a file handle and then a
@@ -1778,6 +1786,9 @@ changes:
   * `encoding` {string|null} **Default:** `null`
   * `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
   * `signal` {AbortSignal} allows aborting an in-progress readFile
+  * `buffer` {Buffer|TypedArray|DataView} A buffer to read into.
+  * `getBuffer` {Function} A synchronous function called with the file size and
+    returning the buffer to read into.
 * Returns: {Promise}  Fulfills with the contents of the file.
 
 Asynchronously reads the entire contents of a file.
@@ -1787,6 +1798,11 @@ as a {Buffer} object. Otherwise, the data will be a string.
 
 If `options` is a string, then it specifies the encoding.
 
+The `buffer` and `getBuffer` options cannot be used together. If one of them is
+provided and no encoding is specified, the returned {Buffer} is a view over the
+supplied buffer containing only the bytes read. If the supplied buffer is too
+small to contain the entire file, the promise will be rejected.
+
 When the `path` is a directory, the behavior of `fsPromises.readFile()` is
 platform-specific. On macOS, Linux, and Windows, the promise will be rejected
 with an error. On FreeBSD, a representation of the directory's contents will be
@@ -4266,6 +4282,9 @@ changes:
   * `encoding` {string|null} **Default:** `null`
   * `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
   * `signal` {AbortSignal} allows aborting an in-progress readFile
+  * `buffer` {Buffer|TypedArray|DataView} A buffer to read into.
+  * `getBuffer` {Function} A synchronous function called with the file size and
+    returning the buffer to read into.
 * `callback` {Function}
   * `err` {Error|AggregateError}
   * `data` {string|Buffer}
@@ -4286,6 +4305,11 @@ contents of the file.
 
 If no encoding is specified, then the raw buffer is returned.
 
+The `buffer` and `getBuffer` options cannot be used together. If one of them is
+provided and no encoding is specified, the returned {Buffer} is a view over the
+supplied buffer containing only the bytes read. If the supplied buffer is too
+small to contain the entire file, the callback is called with an error.
+
 If `options` is a string, then it specifies the encoding:
 
 ```mjs
@@ -6441,6 +6465,9 @@ changes:
 * `options` {Object|string}
   * `encoding` {string|null} **Default:** `null`
   * `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`.
+  * `buffer` {Buffer|TypedArray|DataView} A buffer to read into.
+  * `getBuffer` {Function} A synchronous function called with the file size and
+    returning the buffer to read into.
 * Returns: {string|Buffer}
 
 Returns the contents of the `path`.
@@ -6451,6 +6478,11 @@ this API: [`fs.readFile()`][].
 If the `encoding` option is specified then this function returns a
 string. Otherwise it returns a buffer.
 
+The `buffer` and `getBuffer` options cannot be used together. If one of them is
+provided and no encoding is specified, the returned {Buffer} is a view over the
+supplied buffer containing only the bytes read. If the supplied buffer is too
+small to contain the entire file, an error will be thrown.
+
 Similar to [`fs.readFile()`][], when the path is a directory, the behavior of
 `fs.readFileSync()` is platform-specific.
 

lib/fs.js

@@ -111,6 +111,8 @@ const {
   handleErrorFromBinding,
   preprocessSymlinkDestination,
   Stats,
+  getReadFileBuffer,
+  getReadFileBufferByteLengthName,
   getStatFsFromBinding,
   getStatsFromBinding,
   realpathCacheKey,
@@ -123,6 +125,7 @@ const {
   validateOffsetLengthWrite,
   validatePath,
   validatePosition,
+  validateReadFileBufferOptions,
   validateRmOptions,
   validateRmOptionsSync,
   validateRmdirOptions,
@@ -319,13 +322,7 @@ function readFileAfterStat(err, stats) {
   }
 
   try {
-    if (size === 0) {
-      // TODO(BridgeAR): If an encoding is set, use the StringDecoder to concat
-      // the result and reuse the buffer instead of allocating a new one.
-      context.buffers = [];
-    } else {
-      context.buffer = Buffer.allocUnsafeSlow(size);
-    }
+    context.prepare();
   } catch (err) {
     return context.close(err);
   }
@@ -358,8 +355,9 @@ function readFile(path, options, callback) {
   callback ||= options;
   validateFunction(callback, 'cb');
   options = getOptions(options, { flag: 'r' });
+  validateReadFileBufferOptions(options);
   ReadFileContext ??= require('internal/fs/read/context');
-  const context = new ReadFileContext(callback, options.encoding);
+  const context = new ReadFileContext(callback, options);
   context.isUserFd = isFd(path); // File descriptor ownership
 
   if (options.signal) {
@@ -405,6 +403,18 @@ function tryCreateBuffer(size, fd, isUserFd) {
   return buffer;
 }
 
+function tryGetReadFileBuffer(options, size, fd, isUserFd) {
+  let threw = true;
+  let buffer;
+  try {
+    buffer = getReadFileBuffer(options, size);
+    threw = false;
+  } finally {
+    if (threw && !isUserFd) fs.closeSync(fd);
+  }
+  return buffer;
+}
+
 function tryReadSync(fd, isUserFd, buffer, pos, len) {
   let threw = true;
   let bytesRead;
@@ -417,6 +427,36 @@ function tryReadSync(fd, isUserFd, buffer, pos, len) {
   return bytesRead;
 }
 
+function tryReadSyncWithUserBuffer(fd, isUserFd, buffer, byteLengthName) {
+  let pos = 0;
+  let bytesRead = 0;
+
+  while (pos < buffer.byteLength) {
+    bytesRead = tryReadSync(fd, isUserFd, buffer, pos, buffer.byteLength - pos);
+    pos += bytesRead;
+
+    if (bytesRead === 0) {
+      return pos;
+    }
+  }
+
+  const extraBuffer = tryCreateBuffer(1, fd, isUserFd);
+  bytesRead = tryReadSync(fd, isUserFd, extraBuffer, 0, 1);
+
+  if (bytesRead !== 0) {
+    if (!isUserFd) {
+      fs.closeSync(fd);
+    }
+    throw new ERR_INVALID_ARG_VALUE(
+      byteLengthName,
+      buffer.byteLength,
+      'is too small to contain the entire file',
+    );
+  }
+
+  return pos;
+}
+
 /**
  * Synchronously reads the entire contents of a file.
  * @param {string | Buffer | URL | number} path
@@ -428,8 +468,12 @@ function tryReadSync(fd, isUserFd, buffer, pos, len) {
  */
 function readFileSync(path, options) {
   options = getOptions(options, { flag: 'r' });
+  validateReadFileBufferOptions(options);
+  const hasUserBuffer =
+    options.buffer !== undefined || options.getBuffer !== undefined;
 
-  if (options.encoding === 'utf8' || options.encoding === 'utf-8') {
+  if ((options.encoding === 'utf8' || options.encoding === 'utf-8') &&
+      !hasUserBuffer) {
     if (!isInt32(path)) {
       path = getValidatedPath(path);
     }
@@ -445,15 +489,31 @@ function readFileSync(path, options) {
   let buffer; // Single buffer with file data
   let buffers; // List for when size is unknown
 
-  if (size === 0) {
+  if (hasUserBuffer) {
+    buffer = tryGetReadFileBuffer(options, size, fd, isUserFd);
+  } else if (size === 0) {
     buffers = [];
   } else {
     buffer = tryCreateBuffer(size, fd, isUserFd);
   }
 
   let bytesRead;
 
-  if (size !== 0) {
+  if (hasUserBuffer) {
+    if (size !== 0) {
+      do {
+        bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
+        pos += bytesRead;
+      } while (bytesRead !== 0 && pos < size);
+    } else {
+      pos = tryReadSyncWithUserBuffer(
+        fd,
+        isUserFd,
+        buffer,
+        getReadFileBufferByteLengthName(options),
+      );
+    }
+  } else if (size !== 0) {
     do {
       bytesRead = tryReadSync(fd, isUserFd, buffer, pos, size - pos);
       pos += bytesRead;
@@ -474,7 +534,9 @@ function readFileSync(path, options) {
   if (!isUserFd)
     fs.closeSync(fd);
 
-  if (size === 0) {
+  if (hasUserBuffer) {
+    buffer = buffer.subarray(0, pos);
+  } else if (size === 0) {
     // Data was collected into the buffers list.
     buffer = Buffer.concat(buffers, pos);
   } else if (pos < size) {

lib/internal/fs/promises.js

@@ -66,6 +66,8 @@ const {
   getStatFsFromBinding,
   getStatsFromBinding,
   getValidatedPath,
+  getReadFileBuffer,
+  getReadFileBufferByteLengthName,
   preprocessSymlinkDestination,
   stringToFlags,
   stringToSymlinkType,
@@ -76,6 +78,7 @@ const {
   validateOffsetLengthRead,
   validateOffsetLengthWrite,
   validatePosition,
+  validateReadFileBufferOptions,
   validateRmOptions,
   validateRmdirOptions,
   validateStringAfterArrayBufferView,
@@ -1157,6 +1160,56 @@ async function writeFileHandle(filehandle, data, signal, encoding) {
   } while (remaining > 0);
 }
 
+async function readFileHandleWithUserBuffer(filehandle, options, size) {
+  const signal = options?.signal;
+  const encoding = options?.encoding;
+  const buffer = getReadFileBuffer(options, size);
+  const byteLengthName = getReadFileBufferByteLengthName(options);
+  let totalRead = 0;
+
+  while (totalRead < buffer.byteLength) {
+    checkAborted(signal);
+
+    const length = size === 0 ?
+      buffer.byteLength - totalRead :
+      MathMin(size - totalRead, kReadFileBufferLength);
+
+    const bytesRead = (await PromisePrototypeThen(
+      binding.read(filehandle.fd, buffer, totalRead, length, -1, kUsePromises),
+      undefined,
+      handleErrorFromBinding,
+    )) ?? 0;
+
+    totalRead += bytesRead;
+
+    if (bytesRead === 0 || totalRead === size) {
+      const result = buffer.subarray(0, totalRead);
+      return encoding ? result.toString(encoding) : result;
+    }
+  }
+
+  if (size === 0) {
+    checkAborted(signal);
+
+    const extraBuffer = Buffer.allocUnsafeSlow(1);
+    const bytesRead = (await PromisePrototypeThen(
+      binding.read(filehandle.fd, extraBuffer, 0, 1, -1, kUsePromises),
+      undefined,
+      handleErrorFromBinding,
+    )) ?? 0;
+
+    if (bytesRead !== 0) {
+      throw new ERR_INVALID_ARG_VALUE(
+        byteLengthName,
+        buffer.byteLength,
+        'is too small to contain the entire file',
+      );
+    }
+  }
+
+  return encoding ? buffer.toString(encoding) : buffer.subarray(0, totalRead);
+}
+
 async function readFileHandle(filehandle, options) {
   const signal = options?.signal;
   const encoding = options?.encoding;
@@ -1185,6 +1238,10 @@ async function readFileHandle(filehandle, options) {
   if (size > kIoMaxLength)
     throw new ERR_FS_FILE_TOO_LARGE(size);
 
+  if (options.buffer !== undefined || options.getBuffer !== undefined) {
+    return readFileHandleWithUserBuffer(filehandle, options, size);
+  }
+
   let totalRead = 0;
   const noSize = size === 0;
   let buffer = Buffer.allocUnsafeSlow(length);
@@ -1925,6 +1982,7 @@ async function appendFile(path, data, options) {
 
 async function readFile(path, options) {
   options = getOptions(options, { flag: 'r' });
+  validateReadFileBufferOptions(options);
   const flag = options.flag || 'r';
 
   if (path instanceof FileHandle)