Add PreserveDirectories option to ScriptExecutor to retain log directory structure on upload

VERSION

@@ -1 +1 @@
-3.3.14
+3.3.15

src/VirtualClient/VirtualClient.Actions.UnitTests/ScriptExecutor/ScriptExecutorTests.cs

@@ -11,9 +11,11 @@ namespace VirtualClient.Actions
     using System.Text.RegularExpressions;
     using System.Threading;
     using System.Threading.Tasks;
+    using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.DependencyInjection.Extensions;
     using Moq;
     using NUnit.Framework;
+    using VirtualClient.Common.Contracts;
     using VirtualClient.Common.Telemetry;
     using VirtualClient.Contracts;
 
@@ -380,6 +382,147 @@ public void ScriptExecutorMovesTheLogFilesToCorrectDirectory_Unix(PlatformID pla
             }
         }
 
+        [Test]
+        [TestCase(PlatformID.Win32NT)]
+        [TestCase(PlatformID.Unix)]
+        public void ScriptExecutorPreservesTheLogDirectoryStructureOnDiskWhenRequested(PlatformID platform)
+        {
+            this.SetupTest(platform);
+            this.mockFixture.Parameters[nameof(ScriptExecutor.LogPaths)] = "*.log";
+            this.mockFixture.Parameters[nameof(ScriptExecutor.PreserveDirectories)] = true;
+
+            // A log file that exists within a sub-directory of the script/executable directory.
+            string nestedLogFile = "subfolder/file.log";
+            bool logDirectoryStructurePreserved = false;
+
+            // The relative path (from the script directory) that should be retained under the central logs directory.
+            this.mockFixture.FileSystem.Setup(fe => fe.Path.GetRelativePath(It.IsAny<string>(), It.IsAny<string>()))
+                .Returns(nestedLogFile);
+
+            // Only the file-name matching path (search pattern '*.log') should return the nested log file.
+            this.mockFixture.FileSystem.Setup(fe => fe.Directory.GetFiles(It.IsAny<string>(), "*.log", SearchOption.AllDirectories))
+                .Returns(new[] { nestedLogFile });
+
+            using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture))
+            {
+                this.mockFixture.File.Setup(fe => fe.Move(It.IsAny<string>(), It.IsAny<string>(), true))
+                    .Callback<string, string, bool>((sourcePath, destinitionPath, overwrite) =>
+                    {
+                        if (sourcePath.EndsWith("file.log"))
+                        {
+                            logDirectoryStructurePreserved = destinitionPath.Contains("subfolder");
+                        }
+                    });
+
+                this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => this.mockFixture.Process;
+
+                Assert.DoesNotThrowAsync(() => executor.ExecuteAsync(CancellationToken.None));
+                Assert.IsTrue(logDirectoryStructurePreserved);
+            }
+        }
+
+        [Test]
+        [TestCase(PlatformID.Win32NT)]
+        [TestCase(PlatformID.Unix)]
+        public void ScriptExecutorFlattensTheLogDirectoryStructureOnDiskByDefault(PlatformID platform)
+        {
+            this.SetupTest(platform);
+            this.mockFixture.Parameters[nameof(ScriptExecutor.LogPaths)] = "*.log";
+
+            // A log file that exists within a sub-directory of the script/executable directory.
+            string nestedLogFile = "subfolder/file.log";
+            bool logDirectoryStructurePreserved = true;
+
+            this.mockFixture.FileSystem.Setup(fe => fe.Path.GetRelativePath(It.IsAny<string>(), It.IsAny<string>()))
+                .Returns(nestedLogFile);
+
+            // Only the file-name matching path (search pattern '*.log') should return the nested log file.
+            this.mockFixture.FileSystem.Setup(fe => fe.Directory.GetFiles(It.IsAny<string>(), "*.log", SearchOption.AllDirectories))
+                .Returns(new[] { nestedLogFile });
+
+            using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture))
+            {
+                this.mockFixture.File.Setup(fe => fe.Move(It.IsAny<string>(), It.IsAny<string>(), true))
+                    .Callback<string, string, bool>((sourcePath, destinitionPath, overwrite) =>
+                    {
+                        if (sourcePath.EndsWith("file.log"))
+                        {
+                            logDirectoryStructurePreserved = destinitionPath.Contains("subfolder");
+                        }
+                    });
+
+                this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => this.mockFixture.Process;
+
+                Assert.DoesNotThrowAsync(() => executor.ExecuteAsync(CancellationToken.None));
+                Assert.IsFalse(logDirectoryStructurePreserved);
+            }
+        }
+
+        [Test]
+        [TestCase(PlatformID.Win32NT)]
+        [TestCase(PlatformID.Unix)]
+        public void ScriptExecutorPreservesTheDirectoryStructureInTheUploadPathWhenRequested(PlatformID platform)
+        {
+            this.SetupTest(platform);
+            this.mockFixture.Parameters[nameof(ScriptExecutor.PreserveDirectories)] = true;
+
+            // A content store must exist for file uploads to be requested.
+            this.mockFixture.Dependencies.AddSingleton<IEnumerable<IBlobManager>>(new List<IBlobManager> { this.mockFixture.ContentBlobManager.Object });
+
+            // A log file that was moved into a subfolder within the central logs directory.
+            string logsRootDirectory = this.mockFixture.Combine(this.mockFixture.PlatformSpecifics.LogsDirectory, "generictool");
+            string movedLogFile = this.mockFixture.Combine(logsRootDirectory, "subfolder", "file.log");
+            FileUploadDescriptor capturedDescriptor = null;
+
+            // The upload request writes the descriptor as JSON. Capture it to inspect the blob path.
+            this.mockFixture.File.Setup(fe => fe.WriteAllTextAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+                .Callback<string, string, CancellationToken>((path, contents, token) =>
+                {
+                    capturedDescriptor = contents.FromJson<FileUploadDescriptor>();
+                })
+                .Returns(Task.CompletedTask);
+
+            using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture))
+            {
+                Assert.DoesNotThrowAsync(() => executor.RequestLogUploadsAsync(new[] { movedLogFile }, logsRootDirectory));
+                Assert.IsNotNull(capturedDescriptor);
+                Assert.IsTrue(
+                    capturedDescriptor.BlobPath.Replace('\\', '/').Contains("/subfolder/"),
+                    $"Expected the upload blob path to preserve the 'subfolder' directory but was '{capturedDescriptor.BlobPath}'.");
+            }
+        }
+
+        [Test]
+        [TestCase(PlatformID.Win32NT)]
+        [TestCase(PlatformID.Unix)]
+        public void ScriptExecutorFlattensTheDirectoryStructureInTheUploadPathByDefault(PlatformID platform)
+        {
+            this.SetupTest(platform);
+
+            // A content store must exist for file uploads to be requested.
+            this.mockFixture.Dependencies.AddSingleton<IEnumerable<IBlobManager>>(new List<IBlobManager> { this.mockFixture.ContentBlobManager.Object });
+
+            string logsRootDirectory = this.mockFixture.Combine(this.mockFixture.PlatformSpecifics.LogsDirectory, "generictool");
+            string movedLogFile = this.mockFixture.Combine(logsRootDirectory, "subfolder", "file.log");
+            FileUploadDescriptor capturedDescriptor = null;
+
+            this.mockFixture.File.Setup(fe => fe.WriteAllTextAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+                .Callback<string, string, CancellationToken>((path, contents, token) =>
+                {
+                    capturedDescriptor = contents.FromJson<FileUploadDescriptor>();
+                })
+                .Returns(Task.CompletedTask);
+
+            using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture))
+            {
+                Assert.DoesNotThrowAsync(() => executor.RequestLogUploadsAsync(new[] { movedLogFile }, logsRootDirectory));
+                Assert.IsNotNull(capturedDescriptor);
+                Assert.IsFalse(
+                    capturedDescriptor.BlobPath.Replace('\\', '/').Contains("/subfolder/"),
+                    $"Expected the upload blob path to be flattened but was '{capturedDescriptor.BlobPath}'.");
+            }
+        }
+
         [Test]
         [TestCase(PlatformID.Win32NT)]
         [TestCase(PlatformID.Unix)]
@@ -459,6 +602,11 @@ public TestScriptExecutor(MockFixture fixture)
             {
                 return base.ExecuteAsync(telemetryContext, cancellationToken);
             }
+
+            public new Task RequestLogUploadsAsync(IEnumerable<string> logPaths, string logsRootDirectory = null)
+            {
+                return base.RequestLogUploadsAsync(logPaths, logsRootDirectory);
+            }
         }
     }
 }

src/VirtualClient/VirtualClient.Actions/ScriptExecutor/ScriptExecutor.cs

@@ -111,6 +111,25 @@ public bool RunElevated
             }
         }
 
+        /// <summary>
+        /// True to preserve the directory structure of the script-generated log files (relative to the
+        /// script directory) both on the file system and within the content/blob store virtual paths
+        /// when uploaded. False to flatten all matched log files into the central logs directory root.
+        /// Default = false.
+        /// </summary>
+        public bool PreserveDirectories
+        {
+            get
+            {
+                return this.Parameters.GetValue<bool>(nameof(this.PreserveDirectories), false);
+            }
+
+            set
+            {
+                this.Parameters[nameof(this.PreserveDirectories)] = value;
+            }
+        }
+
         /// <summary>
         /// The full path to the script executable.
         /// </summary>
@@ -313,15 +332,20 @@ protected async Task CaptureLogsAsync(CancellationToken cancellationToken)
                         foreach (string logFilePath in this.fileSystem.Directory.GetFiles(fullLogPath, "*", SearchOption.AllDirectories))
                         {
                             var logs = await this.MoveLogsAsync(logFilePath, destinationLogsDir, cancellationToken, sourceRootDirectory: fullLogPath);
-                            await this.RequestLogUploadsAsync(logs);
+                            await this.RequestLogUploadsAsync(logs, destinationLogsDir);
                         }
                     }
 
                     // Check for Matching FileNames
                     foreach (string logFilePath in this.fileSystem.Directory.GetFiles(this.ExecutableDirectory, logPath, SearchOption.AllDirectories))
                     {
-                        var logs = await this.MoveLogsAsync(logFilePath, destinationLogsDir, cancellationToken);
-                        await this.RequestLogUploadsAsync(logs);
+                        var logs = await this.MoveLogsAsync(
+                            logFilePath,
+                            destinationLogsDir,
+                            cancellationToken,
+                            sourceRootDirectory: this.PreserveDirectories ? this.ExecutableDirectory : null);
+
+                        await this.RequestLogUploadsAsync(logs, destinationLogsDir);
                     }
                 }
             }
@@ -330,19 +354,37 @@ protected async Task CaptureLogsAsync(CancellationToken cancellationToken)
             if (this.fileSystem.File.Exists(this.MetricsFilePath))
             {
                 var logs = await this.MoveLogsAsync(this.MetricsFilePath, destinationLogsDir, cancellationToken);
-                await this.RequestLogUploadsAsync(logs);
+                await this.RequestLogUploadsAsync(logs, destinationLogsDir);
             }
         }
 
         /// <summary>
         /// Requests a file upload for each of the log file paths provided.
         /// </summary>
-        protected async Task RequestLogUploadsAsync(IEnumerable<string> logPaths)
+        /// <param name="logPaths">The set of (already moved) log file paths to upload.</param>
+        /// <param name="logsRootDirectory">
+        /// The central logs directory the files were moved into. When <see cref="PreserveDirectories"/> is true, the
+        /// relative subdirectory of each file (under this root) is preserved in the content/blob store virtual path.
+        /// </param>
+        protected async Task RequestLogUploadsAsync(IEnumerable<string> logPaths, string logsRootDirectory = null)
         {
             if (logPaths?.Any() == true && this.TryGetContentStoreManager(out IBlobManager blobManager))
             {
                 foreach (string logPath in logPaths)
                 {
+                    string subPath = null;
+
+                    if (this.PreserveDirectories && !string.IsNullOrWhiteSpace(logsRootDirectory))
+                    {
+                        // Preserve the relative subdirectory (under the central logs directory) in the
+                        // blob/content store virtual path so the uploaded logs retain the directory structure.
+                        string relativeSubdirectory = this.fileSystem.GetRelativeSubdirectory(logsRootDirectory, logPath);
+                        if (!string.IsNullOrWhiteSpace(relativeSubdirectory))
+                        {
+                            subPath = relativeSubdirectory;
+                        }
+                    }
+
                     FileUploadDescriptor descriptor = this.CreateFileUploadDescriptor(
                         new FileContext(
                             this.fileSystem.FileInfo.New(logPath),
@@ -353,7 +395,8 @@ protected async Task RequestLogUploadsAsync(IEnumerable<string> logPaths)
                             this.ToolName,
                             this.Scenario,
                             null,
-                            this.Roles?.FirstOrDefault()));
+                            this.Roles?.FirstOrDefault()),
+                        subPath: subPath);
 
                     await this.RequestFileUploadAsync(descriptor);
                 }

src/VirtualClient/VirtualClient.Main/profiles/EXAMPLE-EXECUTE-SCRIPT.json

@@ -13,7 +13,8 @@
         "PackageName": "exampleWorkload",
         "WorkloadPackage": "exampleworkload.1.1.0.zip",
         "FailFast": false,
-        "RunElevated": false
+        "RunElevated": false,
+        "PreserveDirectories": false
     },
     "Actions": [
         {
@@ -27,6 +28,7 @@
                 "PackageName": "$.Parameters.PackageName",
                 "FailFast": "$.Parameters.FailFast",
                 "RunElevated": "$.Parameters.RunElevated",
+                "PreserveDirectories": "$.Parameters.PreserveDirectories",
                 "Tags": "Test,VC,Script"
             }
         }