feat(storage): migrate transfer manager samples and tests

storage/downloadFileInChunksWithTransferManager.js

@@ -0,0 +1,82 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+'use strict';
+
+const path = require('path');
+const cwd = path.join(__dirname, '..');
+
+// sample-metadata:
+//   title: Download a File in Chunks With Transfer Manager
+//   description: Downloads a single file in in chunks in parallel utilizing transfer manager.
+//   usage: node downloadFileInChunksWithTransferManager.js <BUCKET_NAME> <FILE_NAME> <DESTINATION_FILE_NAME> <CHUNK_SIZE>
+
+function main(
+  bucketName = 'my-bucket',
+  fileName = 'file1.txt',
+  destFileName = path.join(cwd, fileName),
+  chunkSize = 1024
+) {
+  // [START storage_transfer_manager_download_chunks_concurrently]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The ID of the GCS file to download
+  // const fileName = 'your-file-name';
+
+  // The path to which the file should be downloaded
+  // const destFileName = '/local/path/to/file.txt';
+
+  // The size of each chunk to be downloaded
+  // const chunkSize = 1024;
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager client
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function downloadFileInChunksWithTransferManager() {
+    try {
+      // Downloads the files
+      await transferManager.downloadFileInChunks(fileName, {
+        destination: destFileName,
+        chunkSizeBytes: chunkSize,
+      });
+
+      console.log(
+        `gs://${bucketName}/${fileName} downloaded to ${destFileName}.`
+      );
+    } catch (error) {
+      console.error(
+        'Error downloading file in chunks with Transfer Manager:',
+        error.message || error
+      );
+    }
+  }
+
+  downloadFileInChunksWithTransferManager();
+  // [END storage_transfer_manager_download_chunks_concurrently]
+}
+
+main(...process.argv.slice(2));

storage/downloadFolderWithTransferManager.js

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+'use strict';
+
+// sample-metadata:
+//   title: Download Folder With Transfer Manager
+//   description: Downloads a folder in parallel utilizing transfer manager.
+//   usage: node downloadFolderWithTransferManager.js <BUCKET_NAME> <FOLDER_NAME>
+
+function main(bucketName = 'my-bucket', folderName = 'my-folder') {
+  // [START storage_transfer_manager_download_folder]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The ID of the GCS folder to download. The folder will be downloaded to the local path of the executing code.
+  // const folderName = 'your-folder-name';
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager client
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function downloadFolderWithTransferManager() {
+    try {
+      // Downloads the folder
+      await transferManager.downloadManyFiles(folderName);
+
+      console.log(
+        `gs://${bucketName}/${folderName} downloaded to ${folderName}.`
+      );
+    } catch (error) {
+      console.error(
+        'Error downloading folder with Transfer Manager:',
+        error.message || error
+      );
+    }
+  }
+
+  downloadFolderWithTransferManager();
+  // [END storage_transfer_manager_download_folder]
+}
+
+main(...process.argv.slice(2));

storage/downloadManyFilesWithTransferManager.js

@@ -0,0 +1,74 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+'use strict';
+
+// sample-metadata:
+//   title: Download Many Files With Transfer Manager
+//   description: Downloads many files in parallel utilizing transfer manager.
+//   usage: node downloadManyFilesWithTransferManager.js <BUCKET_NAME> <FIRST_FILE_NAME> <SECOND_FILE_NAME>
+
+function main(
+  bucketName = 'my-bucket',
+  firstFileName = 'file1.txt',
+  secondFileName = 'file2.txt'
+) {
+  // [START storage_transfer_manager_download_many]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The ID of the first GCS file to download
+  // const firstFileName = 'your-first-file-name';
+
+  // The ID of the second GCS file to download
+  // const secondFileName = 'your-second-file-name;
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager client
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function downloadManyFilesWithTransferManager() {
+    try {
+      // Downloads the files
+      await transferManager.downloadManyFiles([firstFileName, secondFileName]);
+
+      for (const fileName of [firstFileName, secondFileName]) {
+        console.log(
+          `gs://${bucketName}/${fileName} downloaded to ${fileName}.`
+        );
+      }
+    } catch (error) {
+      console.error(
+        'Error downloading files with Transfer Manager:',
+        error.message || error
+      );
+    }
+  }
+
+  downloadManyFilesWithTransferManager();
+  // [END storage_transfer_manager_download_many]
+}
+
+main(...process.argv.slice(2));

storage/system-test/transfer-manager.test.js

@@ -0,0 +1,118 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const path = require('path');
+const {Storage} = require('@google-cloud/storage');
+const {before, after, it, describe} = require('mocha');
+const uuid = require('uuid');
+const cp = require('child_process');
+const {assert} = require('chai');
+
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+const storage = new Storage();
+const cwd = path.join(__dirname, '..');
+const bucketName = generateName();
+const bucket = storage.bucket(bucketName);
+const firstFileName = 'test.txt';
+const secondFileName = 'test2.txt';
+const resourcesPath = path.join(cwd, 'resources');
+const firstFilePath = path.join(resourcesPath, firstFileName);
+const secondFilePath = path.join(resourcesPath, secondFileName);
+const downloadFilePath = path.join(cwd, 'downloaded.txt');
+const chunkSize = 1024;
+
+describe('transfer manager', () => {
+  before(async () => {
+    await bucket.create();
+  });
+
+  after(async () => {
+    await bucket.deleteFiles({force: true}).catch(console.error);
+    await bucket.delete().catch(console.error);
+  });
+
+  it('should upload multiple files', async () => {
+    const output = execSync(
+      `node uploadManyFilesWithTransferManager.js ${bucketName} ${firstFilePath} ${secondFilePath}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `${firstFilePath} uploaded to ${bucketName}.\n${secondFilePath} uploaded to ${bucketName}`
+      )
+    );
+  });
+
+  it('should download multiple files', async () => {
+    const output = execSync(
+      `node downloadManyFilesWithTransferManager.js ${bucketName} ${firstFilePath} ${secondFilePath}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `gs://${bucketName}/${firstFilePath} downloaded to ${firstFilePath}.\ngs://${bucketName}/${secondFilePath} downloaded to ${secondFilePath}.`
+      )
+    );
+  });
+
+  it('should download a file utilizing chunked download', async () => {
+    const output = execSync(
+      `node downloadFileInChunksWithTransferManager.js ${bucketName} ${firstFilePath} ${downloadFilePath} ${chunkSize}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `gs://${bucketName}/${firstFilePath} downloaded to ${downloadFilePath}.`
+      )
+    );
+  });
+
+  it('should upload a file utilizing chunked upload', async () => {
+    const output = execSync(
+      `node uploadFileInChunksWithTransferManager.js ${bucketName} ${firstFilePath} ${chunkSize}`
+    );
+    assert.match(
+      output,
+      new RegExp(`${firstFilePath} uploaded to ${bucketName}.`)
+    );
+  });
+
+  it('should upload a directory', async () => {
+    const output = execSync(
+      `node uploadDirectoryWithTransferManager.js ${bucketName} ${resourcesPath}`
+    );
+    assert.match(
+      output,
+      new RegExp(`${resourcesPath} uploaded to ${bucketName}.`)
+    );
+  });
+
+  it('should download a directory', async () => {
+    const output = execSync(
+      `node downloadFolderWithTransferManager.js ${bucketName} ${resourcesPath}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `gs://${bucketName}/${resourcesPath} downloaded to ${resourcesPath}.`
+      )
+    );
+  });
+});
+
+function generateName() {
+  return `nodejs-storage-samples-${uuid.v4()}`;
+}