From 253187dc990c08d0c080e94818d6f18071b57a84 Mon Sep 17 00:00:00 2001 From: Erica Vellanoweth Date: Fri, 20 Mar 2026 09:50:46 -0700 Subject: [PATCH 1/2] first commit --- .../HammerDB/HammerDBClientExecutor.cs | 4 +- .../HammerDB/HammerDBExecutor.cs | 87 +------------------ .../Sysbench/SysbenchConfiguration.cs | 47 +--------- .../Sysbench/SysbenchExecutor.cs | 35 +------- .../MySQLServerConfigurationTests.cs | 54 ------------ .../PostgreSQLServerConfigurationTests.cs | 54 ------------ .../MySqlServer/MySqlServerConfiguration.cs | 30 ------- .../PostgreSQLServerConfiguration.cs | 75 ---------------- .../profiles/PERF-MYSQL-SYSBENCH-OLTP.json | 44 +++------- .../profiles/PERF-MYSQL-SYSBENCH-TPCC.json | 44 +++------- .../PERF-POSTGRESQL-HAMMERDB-TPCC.json | 2 +- .../PERF-POSTGRESQL-HAMMERDB-TPCH.json | 2 +- .../PERF-POSTGRESQL-SYSBENCH-OLTP.json | 2 +- .../PERF-POSTGRESQL-SYSBENCH-TPCC.json | 2 +- 14 files changed, 34 insertions(+), 448 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs index 579821edfe..4962cbca24 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs @@ -21,6 +21,8 @@ namespace VirtualClient.Actions /// public class HammerDBClientExecutor : HammerDBExecutor { + private static string runTransactionsTclName = "runTransactions.tcl"; + /// /// Initializes a new instance of the class. /// @@ -154,7 +156,7 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok using (IProcessProxy process = await this.ExecuteCommandAsync( command, - $"{script} --runTransactionsTCLFilePath {this.RunTransactionsTclName}", + $"{script} --runTransactionsTCLFilePath {runTransactionsTclName}", this.HammerDBPackagePath, telemetryContext, cancellationToken)) diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs index 296d794808..530dbb5d7f 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs @@ -25,6 +25,7 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-x64")] public class HammerDBExecutor : VirtualClientComponent { + private static string createDBTclName = "createDB.tcl"; private readonly IStateManager stateManager; private static readonly List Factors = new List { 1, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 100000 }; @@ -57,28 +58,6 @@ public string Action } } - /// - /// Defines the name of the createDB TCL file. - /// - public string CreateDBTclName - { - get - { - return "createDB.tcl"; - } - } - - /// - /// Defines the name of the runTransactions TCL file. - /// - public string RunTransactionsTclName - { - get - { - return "runTransactions.tcl"; - } - } - /// /// Defines the name of the PostgreSQL database to create/use for the transactions. /// @@ -137,18 +116,6 @@ public string WarehouseCount } } - /// - /// Disk filter specified - /// - public string DiskFilter - { - get - { - // and 256G - return this.Parameters.GetValue(nameof(this.DiskFilter), "osdisk:false&sizegreaterthan:256g"); - } - } - /// /// Workload duration. /// @@ -341,7 +308,7 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C private async Task PrepareSQLDatabase(EventContext telemetryContext, CancellationToken cancellationToken) { string command = "python3"; - string arguments = $"{this.PlatformSpecifics.Combine(this.HammerDBPackagePath, "populate-database.py")} --createDBTCLPath {this.CreateDBTclName}"; + string arguments = $"{this.PlatformSpecifics.Combine(this.HammerDBPackagePath, "populate-database.py")} --createDBTCLPath {createDBTclName}"; using (IProcessProxy process = await this.ExecuteCommandAsync( command, @@ -383,12 +350,6 @@ private async Task GenerateCommandLineArguments(CancellationToken cancellationTo string arguments = $"{this.PlatformSpecifics.Combine(this.HammerDBPackagePath, "configure-workload-generator.py")} --workload {this.Workload} --sqlServer {this.SQLServer} --port {this.Port}" + $" --virtualUsers {this.VirtualUsers} --password {this.SuperUserPassword} --dbName {this.DatabaseName} --hostIPAddress {this.ServerIpAddress}"; - if (this.IsMultiRoleLayout() && this.GetLayoutClientInstance().Role == ClientRole.Server) - { - string directories = await this.GetDataDirectoriesAsync(cancellationToken); - arguments = $"{arguments} --directories {directories}"; - } - if (this.Workload.Equals("tpcc", StringComparison.OrdinalIgnoreCase)) { arguments = $"{arguments} --warehouseCount {this.WarehouseCount} --duration {this.Duration.TotalMinutes}"; @@ -409,50 +370,6 @@ private async Task GenerateCommandLineArguments(CancellationToken cancellationTo this.HammerDBScenarioArguments = arguments; } - private async Task GetDataDirectoriesAsync(CancellationToken cancellationToken) - { - string diskPaths = string.Empty; - - if (!cancellationToken.IsCancellationRequested) - { - IEnumerable disks = await this.SystemManager.DiskManager.GetDisksAsync(cancellationToken) - .ConfigureAwait(false); - - if (disks?.Any() != true) - { - throw new WorkloadException( - "Unexpected scenario. The disks defined for the system could not be properly enumerated.", - ErrorReason.WorkloadUnexpectedAnomaly); - } - - IEnumerable disksToTest = DiskFilters.FilterDisks(disks, this.DiskFilter, this.Platform).ToList(); - - if (disksToTest?.Any() != true) - { - throw new WorkloadException( - "Expected disks to test not found. Given the parameters defined for the profile action/step or those passed " + - "in on the command line, the requisite disks do not exist on the system or could not be identified based on the properties " + - "of the existing disks.", - ErrorReason.DependencyNotFound); - } - - foreach (Disk disk in disksToTest) - { - string path = this.Combine(disk.GetPreferredAccessPath(this.Platform), $"{this.SQLServer.ToLower()}"); - - // Create the directory if it doesn't exist - if (!this.SystemManager.FileSystem.Directory.Exists(path)) - { - this.SystemManager.FileSystem.Directory.CreateDirectory(path); - } - - diskPaths += $"{path}:"; - } - } - - return diskPaths; - } - private static Task OpenFirewallPortsAsync(int port, IFirewallManager firewallManager, CancellationToken cancellationToken) { return firewallManager.EnableInboundConnectionsAsync( diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs index 012664d1a3..5ebaa198b5 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs @@ -59,15 +59,12 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel case ConfigurationAction.Cleanup: await this.CleanUpDatabase(telemetryContext, cancellationToken); break; - case ConfigurationAction.CreateTables: - await this.PrepareDatabase(telemetryContext, cancellationToken); - break; case ConfigurationAction.PopulateTables: await this.PopulateDatabase(telemetryContext, cancellationToken); break; default: throw new DependencyException( - $"The specified Sysbench action '{this.Action}' is not supported. Supported actions include: \"{ConfigurationAction.PopulateTables}, {ConfigurationAction.Cleanup}, {ConfigurationAction.CreateTables}\".", + $"The specified Sysbench action '{this.Action}' is not supported. Supported actions include: \"{ConfigurationAction.PopulateTables}, {ConfigurationAction.Cleanup}\".", ErrorReason.NotSupported); } } @@ -106,41 +103,6 @@ private async Task CleanUpDatabase(EventContext telemetryContext, CancellationTo await this.stateManager.SaveStateAsync(nameof(SysbenchState), state, cancellationToken); } - private async Task PrepareDatabase(EventContext telemetryContext, CancellationToken cancellationToken) - { - SysbenchState state = await this.stateManager.GetStateAsync(nameof(SysbenchState), cancellationToken) - ?? new SysbenchState(); - - if (!state.DatabasePopulated) - { - string serverIp = (this.IsMultiRoleLayout() && this.IsInRole(ClientRole.Client)) ? this.ServerIpAddress : "localhost"; - string sysbenchPrepareArguments = $"{this.BuildSysbenchLoggingArguments(SysbenchMode.Prepare)} --password {this.SuperUserPassword} --hostIpAddress \"{serverIp}\""; - - string command = $"{this.SysbenchPackagePath}/populate-database.py"; - - using (IProcessProxy process = await this.ExecuteCommandAsync( - SysbenchExecutor.PythonCommand, - $"{command} {sysbenchPrepareArguments}", - this.SysbenchPackagePath, - telemetryContext, - cancellationToken)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true); - process.ThrowIfErrored(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly); - } - } - } - else - { - throw new DependencyException( - $"Database preparation failed. A database has already been populated on the system. Please drop the tables, or run \"{ConfigurationAction.Cleanup}\" Action" + - $"before attempting to create new tables on this database.", - ErrorReason.NotSupported); - } - } - private async Task PopulateDatabase(EventContext telemetryContext, CancellationToken cancellationToken) { SysbenchState state = await this.stateManager.GetStateAsync(nameof(SysbenchState), cancellationToken) @@ -152,7 +114,7 @@ await this.Logger.LogMessageAsync($"{this.TypeName}.PopulateDatabase", telemetry { string serverIp = (this.IsMultiRoleLayout() && this.IsInRole(ClientRole.Client)) ? this.ServerIpAddress : "localhost"; - string sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments(SysbenchMode.Populate); + string sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments(); this.sysbenchPopulationArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword} --hostIpAddress \"{serverIp}\""; string script = $"{this.SysbenchPackagePath}/populate-database.py"; @@ -227,11 +189,6 @@ private void AddPopulationDurationMetric(string arguments, IProcessProxy process /// internal class ConfigurationAction { - /// - /// Initializes the tables on the database. - /// - public const string CreateTables = nameof(CreateTables); - /// /// Creates Database on MySQL server and Users on Server and any Clients. /// diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs index 5703d485e5..a0b06764f6 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs @@ -60,27 +60,6 @@ public SysbenchExecutor(IServiceCollection dependencies, IDictionary - /// Defines the mode in which Sysbench is operating. - /// - protected internal enum SysbenchMode - { - /// - /// Creates the database schema with minimal data. - /// - Prepare, - - /// - /// Populates the database with the full dataset. - /// - Populate, - - /// - /// Runs the benchmark workload. - /// - Run - } - /// /// The benchmark (e.g. OLTP, TPCC). /// @@ -409,7 +388,7 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C /// dbName, databaseSystem, benchmark and tableCount. /// /// - protected string BuildSysbenchLoggingArguments(SysbenchMode mode) + protected string BuildSysbenchLoggingArguments() { int tableCount = GetTableCount(this.DatabaseScenario, this.TableCount, this.Workload); int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads); @@ -419,19 +398,11 @@ protected string BuildSysbenchLoggingArguments(SysbenchMode mode) switch (this.Benchmark) { case BenchmarkName.OLTP: - int recordCount = mode == SysbenchMode.Prepare ? 1 : GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount); + int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount); loggingArguments = $"{loggingArguments} --recordCount {recordCount}"; break; case BenchmarkName.TPCC: - int warehouseEstimate = GetWarehouseCount(this.SystemManager, this.DatabaseScenario, this.WarehouseCount); - int warehouseCount = mode switch - { - SysbenchMode.Prepare => 1, - SysbenchMode.Populate => warehouseEstimate - 1, - SysbenchMode.Run => warehouseEstimate, - _ => warehouseEstimate - }; - + int warehouseCount = GetWarehouseCount(this.SystemManager, this.DatabaseScenario, this.WarehouseCount); loggingArguments = $"{loggingArguments} --warehouses {warehouseCount}"; break; default: diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs index 117a8cd906..3e53289d33 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs @@ -317,60 +317,6 @@ public async Task MySQLConfigurationExecutesTheExpectedProcessForRaiseMaxStateme } } - [Test] - public async Task MySQLConfigurationExecutesTheExpectedProcessForDistributeDatabaseCommand() - { - this.fixture.Parameters["Action"] = "DistributeDatabase"; - this.fixture.Parameters["DatabaseName"] = "mysql-test"; - this.fixture.Parameters["TableCount"] = "10"; - - string[] expectedCommands = - { - $"python3 {this.packagePath}/distribute-database.py --dbName mysql-test --directories \"/home/user/mnt_dev_sdc1/mysql;/home/user/mnt_dev_sdd1/mysql;/home/user/mnt_dev_sde1/mysql\"", - }; - - int commandNumber = 0; - bool commandExecuted = false; - - this.fixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - string expectedCommand = expectedCommands[commandNumber]; - - if (expectedCommand == $"{exe} {arguments}") - { - commandExecuted = true; - } - - Assert.IsTrue(commandExecuted); - commandExecuted = false; - commandNumber++; - - InMemoryProcess process = new InMemoryProcess - { - StartInfo = new ProcessStartInfo - { - FileName = exe, - Arguments = arguments - }, - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - - return process; - }; - - this.fixture.StateManager.OnSaveState((stateId, state) => - { - Assert.IsNotNull(state); - }); - - using (TestMySQLServerConfiguration component = new TestMySQLServerConfiguration(this.fixture)) - { - await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - } - private class TestMySQLServerConfiguration : MySQLServerConfiguration { public TestMySQLServerConfiguration(MockFixture fixture) diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs index 73663a6427..c4b431fa2f 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs @@ -174,60 +174,6 @@ public async Task PostgreSQLConfigurationSkipsDatabaseCreationWhenOneExists(Plat Assert.AreEqual(0, commandsExecuted); } - [Test] - [TestCase(PlatformID.Unix, Architecture.X64)] - [TestCase(PlatformID.Win32NT, Architecture.X64)] - public async Task PostgreSQLServerConfigurationExecutesTheExpectedProcessForDistributeDatabaseCommand(PlatformID platform, Architecture architecture) - { - this.SetupTest(platform, architecture); - this.mockFixture.Parameters["Action"] = "DistributeDatabase"; - string expectedCommand; - - if (platform == PlatformID.Unix) - { - expectedCommand = - $"python3 {this.packagePath}/distribute-database.py " + - $"--dbName hammerdbtest " + - $"--directories \"/home/user/mnt_dev_sdc1/postgresql;/home/user/mnt_dev_sdd1/postgresql;/home/user/mnt_dev_sde1/postgresql;\" " + - $"--password [A-Za-z0-9+/=]+"; - } - else - { - string tempPackagePath = this.packagePath.Replace(@"\", @"\\"); - expectedCommand = $"python3 {tempPackagePath}/distribute-database.py --dbName hammerdbtest --directories \"D:\\\\postgresql;E:\\\\postgresql;F:\\\\postgresql;\" --password [A-Za-z0-9+/=]+"; - } - - this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - string executedCommand = $"{exe} {arguments}"; - Assert.IsTrue(Regex.IsMatch(executedCommand, expectedCommand)); - - InMemoryProcess process = new InMemoryProcess - { - StartInfo = new ProcessStartInfo - { - FileName = exe, - Arguments = arguments - }, - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - - return process; - }; - - this.mockFixture.StateManager.OnSaveState((stateId, state) => - { - Assert.IsNotNull(state); - }); - - using (TestPostgreSQLServerConfiguration component = new TestPostgreSQLServerConfiguration(this.mockFixture)) - { - await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - } - private class TestPostgreSQLServerConfiguration : PostgreSQLServerConfiguration { public TestPostgreSQLServerConfiguration(MockFixture fixture) diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 380a0ba45c..15726bd8f9 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -161,10 +161,6 @@ await this.ConfigureMySQLServerAsync(telemetryContext, cancellationToken) await this.CreateMySQLServerDatabaseAsync(telemetryContext, cancellationToken) .ConfigureAwait(false); break; - case ConfigurationAction.DistributeDatabase: - await this.DistributeMySQLDatabaseAsync(telemetryContext, cancellationToken) - .ConfigureAwait(false); - break; case ConfigurationAction.SetGlobalVariables: await this.SetMySQLGlobalVariableAsync(telemetryContext, cancellationToken) .ConfigureAwait(false); @@ -251,27 +247,6 @@ private async Task SetMySQLGlobalVariableAsync(EventContext telemetryContext, Ca } } - private async Task DistributeMySQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - string innoDbDirs = await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); - - string arguments = $"{this.packageDirectory}/distribute-database.py --dbName {this.DatabaseName} --directories \"{innoDbDirs}\""; - - using (IProcessProxy process = await this.ExecuteCommandAsync( - PythonCommand, - arguments, - Environment.CurrentDirectory, - telemetryContext, - cancellationToken)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "MySQLServerConfiguration", logToFile: true); - process.ThrowIfDependencyInstallationFailed(process.StandardError.ToString()); - } - } - } - private async Task GetMySQLInnodbDirectoriesAsync(CancellationToken cancellationToken) { string diskPaths = string.Empty; @@ -360,11 +335,6 @@ internal class ConfigurationAction /// ie. "MAX_PREPARED_STMT_COUNT=1000000;MAX_CONNECTIONS=1024" /// public const string SetGlobalVariables = nameof(SetGlobalVariables); - - /// - /// Distributes existing database to disks on the system - /// - public const string DistributeDatabase = nameof(DistributeDatabase); } internal class ConfigurationState diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index 3634827e08..9347e89900 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -163,10 +163,6 @@ await this.ConfigurePostgreSQLServerAsync(telemetryContext, cancellationToken) await this.SetupPostgreSQLDatabaseAsync(telemetryContext, cancellationToken) .ConfigureAwait(false); break; - case ConfigurationAction.DistributeDatabase: - await this.DistributePostgreSQLDatabaseAsync(telemetryContext, cancellationToken) - .ConfigureAwait(false); - break; } await this.stateManager.SaveStateAsync(stateId, new ConfigurationState(this.Action), cancellationToken); @@ -216,71 +212,6 @@ private async Task SetupPostgreSQLDatabaseAsync(EventContext telemetryContext, C } } - private async Task DistributePostgreSQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - string innoDbDirs = await this.GetPostgreSQLInnodbDirectoriesAsync(cancellationToken); - - string arguments = $"{this.packageDirectory}/distribute-database.py --dbName {this.DatabaseName} --directories \"{innoDbDirs}\" --password {this.SuperUserPassword}"; - - using (IProcessProxy process = await this.ExecuteCommandAsync( - PythonCommand, - arguments, - Environment.CurrentDirectory, - telemetryContext, - cancellationToken)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "PostgreSQLServerConfiguration", logToFile: true); - process.ThrowIfDependencyInstallationFailed(process.StandardError.ToString()); - } - } - } - - private async Task GetPostgreSQLInnodbDirectoriesAsync(CancellationToken cancellationToken) - { - string diskPaths = string.Empty; - - if (!cancellationToken.IsCancellationRequested) - { - IEnumerable disks = await this.SystemManager.DiskManager.GetDisksAsync(cancellationToken) - .ConfigureAwait(false); - - if (disks?.Any() != true) - { - throw new WorkloadException( - "Unexpected scenario. The disks defined for the system could not be properly enumerated.", - ErrorReason.WorkloadUnexpectedAnomaly); - } - - IEnumerable disksToTest = DiskFilters.FilterDisks(disks, this.DiskFilter, this.Platform).ToList(); - - if (disksToTest?.Any() != true) - { - throw new WorkloadException( - "Expected disks to test not found. Given the parameters defined for the profile action/step or those passed " + - "in on the command line, the requisite disks do not exist on the system or could not be identified based on the properties " + - "of the existing disks.", - ErrorReason.DependencyNotFound); - } - - foreach (Disk disk in disksToTest) - { - string postgresqlPath = this.Combine(disk.GetPreferredAccessPath(this.Platform), "postgresql"); - - // Create the directory if it doesn't exist - if (!this.SystemManager.FileSystem.Directory.Exists(postgresqlPath)) - { - this.SystemManager.FileSystem.Directory.CreateDirectory(postgresqlPath); - } - - diskPaths += $"{postgresqlPath};"; - } - } - - return diskPaths; - } - /// /// Supported PostgreSQL Server configuration actions. /// @@ -295,12 +226,6 @@ internal class ConfigurationAction /// Creates Database on PostgreSQL server and Users on Server and any Clients. /// public const string SetupDatabase = nameof(SetupDatabase); - - /// - /// Distributes existing database to disks on the system - /// - public const string DistributeDatabase = nameof(DistributeDatabase); - } internal class ConfigurationState diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json index 4ee1f21ca3..90fd0851e0 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json @@ -150,6 +150,13 @@ } ], "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages": "python3,python3-pyudev,python3-psutil" + } + }, { "Type": "FormatDisks", "Parameters": { @@ -158,9 +165,9 @@ } }, { - "Type": "MountDisks", + "Type": "StripeDisks", "Parameters": { - "Scenario": "CreateMountPoints", + "Scenario": "StripeAndMountDisks", "Role": "Server" } }, @@ -169,7 +176,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql.8.0.36.rev3.zip", + "BlobName": "mysql-server-8.0.36.rev0.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" @@ -185,13 +192,6 @@ "Extract": true } }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" - } - }, { "Type": "MySQLServerInstallation", "Parameters": { @@ -237,30 +237,6 @@ "Role": "Server" } }, - { - "Type": "SysbenchConfiguration", - "Parameters": { - "Scenario": "PrepareMySQLDatabase", - "Action": "CreateTables", - "DatabaseSystem": "MySQL", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - "DatabaseScenario": "$.Parameters.DatabaseScenario", - "PackageName": "sysbench", - "Role": "Server" - } - }, - { - "Type": "MySQLServerConfiguration", - "Parameters": { - "Scenario": "DistributeMySQLDatabase", - "Action": "DistributeDatabase", - "DiskFilter": "$.Parameters.DiskFilter", - "DatabaseName": "$.Parameters.DatabaseName", - "PackageName": "mysql-server", - "Role": "Server" - } - }, { "Type": "SysbenchConfiguration", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json index f48802a6e2..d616771b37 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json @@ -38,6 +38,13 @@ } ], "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages": "python3,python3-pyudev,python3-psutil" + } + }, { "Type": "FormatDisks", "Parameters": { @@ -46,9 +53,9 @@ } }, { - "Type": "MountDisks", + "Type": "StripeDisks", "Parameters": { - "Scenario": "CreateMountPoints", + "Scenario": "StripeAndMountDisks", "Role": "Server" } }, @@ -57,7 +64,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql.8.0.36.rev3.zip", + "BlobName": "mysql-server-8.0.36.rev0.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" @@ -73,13 +80,6 @@ "Extract": true } }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" - } - }, { "Type": "MySQLServerInstallation", "Parameters": { @@ -125,30 +125,6 @@ "Role": "Server" } }, - { - "Type": "SysbenchConfiguration", - "Parameters": { - "Scenario": "PrepareMySQLDatabase", - "Action": "CreateTables", - "DatabaseSystem": "MySQL", - "Benchmark": "TPCC", - "DatabaseName": "$.Parameters.DatabaseName", - "DatabaseScenario": "$.Parameters.DatabaseScenario", - "PackageName": "sysbench", - "Role": "Server" - } - }, - { - "Type": "MySQLServerConfiguration", - "Parameters": { - "Scenario": "DistributeMySQLDatabase", - "Action": "DistributeDatabase", - "DatabaseName": "$.Parameters.DatabaseName", - "DiskFilter": "$.Parameters.DiskFilter", - "PackageName": "mysql-server", - "Role": "Server" - } - }, { "Type": "SysbenchConfiguration", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json index 51666405d7..da7ec886d3 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json @@ -84,7 +84,7 @@ "Parameters": { "Scenario": "DownloadHammerDBPackage", "BlobContainer": "packages", - "BlobName": "hammerdb.4.12.0.rev3.zip", + "BlobName": "hammerdb.4.12.0.rev4.zip", "PackageName": "hammerdb", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json index da5ec3bde5..e077610d55 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json @@ -84,7 +84,7 @@ "Parameters": { "Scenario": "DownloadHammerDBPackage", "BlobContainer": "packages", - "BlobName": "hammerdb.4.12.0.rev3.zip", + "BlobName": "hammerdb.4.12.0.rev4.zip", "PackageName": "hammerdb", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json index f23278cc60..a4c012ffef 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json @@ -170,7 +170,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLServerPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev3.zip", + "BlobName": "postgresql.14.0.0.rev4.zip", "PackageName": "postgresql", "Extract": true, "Role": "Server" diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json index 093c760126..5de85fd32c 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json @@ -58,7 +58,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLServerPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev3.zip", + "BlobName": "postgresql.14.0.0.rev4.zip", "PackageName": "postgresql", "Extract": true, "Role": "Server" From a01c62fc8076d4d64377c0ac35a6632ed91a684c Mon Sep 17 00:00:00 2001 From: Erica Vellanoweth Date: Mon, 6 Apr 2026 09:32:59 -0700 Subject: [PATCH 2/2] Fix disk detection after LVM striping and update database workload profiles - Add /proc/mounts fallback for raid0 path discovery in MySqlServerConfiguration and PostgreSQLServerConfiguration. After StripeDisks creates an LVM volume, lshw no longer reports the logical volume, causing 'Expected disks not found'. The fallback reads /proc/mounts to find the raid0 mount point. - Search all disks (not just filtered) for raid0 access paths in both MySQL and PostgreSQL server configuration components. - Remove unsupported CreateTables and DistributeDatabase dependency steps from PostgreSQL Sysbench TPCC and OLTP profiles. - Tune HammerDB TPCC WarehouseCount to LogicalCoreCount * 10 and TPCH ScaleFactor to 1 for faster validation runs. - Fix HammerDBExecutor async method missing await warning. - Add copilot-instructions.md for repository development guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 190 ++++ .../Sysbench/SysbenchConfigurationTests.cs | 6 +- .../HammerDB/HammerDBExecutor.cs | 4 +- .../Sysbench/SysbenchClientExecutor.cs | 2 +- .../Components/ExecuteCommandTests.cs | 841 ++++++++++++++++++ .../Components/ExecuteCommand.cs | 317 +++++++ .../MySQLServerConfigurationTests.cs | 4 +- .../PostgreSQLServerConfigurationTests.cs | 2 +- .../StripeDisksTests.cs | 24 + .../MySqlServer/MySqlServerConfiguration.cs | 119 ++- .../PostgreSQLServerConfiguration.cs | 85 +- .../VirtualClient.Dependencies/StripeDisks.cs | 23 +- .../profiles/PERF-MYSQL-SYSBENCH-OLTP.json | 16 +- .../profiles/PERF-MYSQL-SYSBENCH-TPCC.json | 16 +- .../PERF-POSTGRESQL-HAMMERDB-TPCC.json | 28 +- .../PERF-POSTGRESQL-HAMMERDB-TPCH.json | 26 +- .../PERF-POSTGRESQL-SYSBENCH-OLTP.json | 55 +- .../PERF-POSTGRESQL-SYSBENCH-TPCC.json | 57 +- 18 files changed, 1651 insertions(+), 164 deletions(-) create mode 100644 .github/copilot-instructions.md create mode 100644 src/VirtualClient/VirtualClient.Core.UnitTests/Components/ExecuteCommandTests.cs create mode 100644 src/VirtualClient/VirtualClient.Core/Components/ExecuteCommand.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000000..4dc7e0bef1 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,190 @@ +# Copilot Instructions for VirtualClient + +## Build, Test, and Lint + +```bash +# Build the solution (builds AnyCPU with Debug, then publishes per-platform with Release) +build.cmd # all platforms +build.cmd --win-x64 # single platform +build.cmd --linux-x64 --linux-arm64 # multiple platforms + +# Run all unit + functional tests +build-test.cmd + +# Run a single test project +dotnet test -c Debug src\VirtualClient\VirtualClient.Actions.UnitTests\VirtualClient.Actions.UnitTests.csproj --no-restore --no-build --filter "(Category=Unit)" --logger "console;verbosity=normal" + +# Run a single test by name +dotnet test -c Debug src\VirtualClient\VirtualClient.Actions.UnitTests\VirtualClient.Actions.UnitTests.csproj --no-restore --no-build --filter "FullyQualifiedName~CoreMarkExecutorTests.CoreMarkExecutorExecutesTheExpectedCommandInLinux" + +# Build NuGet packages (run after build.cmd) +build-packages.cmd +build-packages.cmd --suffix beta + +# Clean build output +clean.cmd +``` + +The solution must build before running tests (`build.cmd` then `build-test.cmd`). The solution is built with **Debug** configuration to support extensions debugging. Publishing uses **Release**. StyleCop, AsyncFixer, and Roslyn analyzers are enforced at build time — warnings are treated as errors. + +Test categories are `Unit` and `Functional`. The test filter in CI is `(Category=Unit|Category=Functional)`. + +## Architecture + +### Project Dependency Graph + +``` +VirtualClient.Main (Entry point, self-contained EXE) +├── VirtualClient.Actions — 50+ workload executors (benchmarks) +├── VirtualClient.Dependencies — Package/tool installers +├── VirtualClient.Monitors — System monitors (GPU, perf counters, etc.) +├── VirtualClient.Api — REST API (ASP.NET Core) for state/heartbeat/events +└── VirtualClient.Core — Runtime: package/state/process/blob managers + ├── VirtualClient.Contracts — Base classes, interfaces, data contracts + │ └── VirtualClient.Common — Extensions, telemetry primitives, Azure SDK wrappers + └── VirtualClient.Common +``` + +### Component Model + +All actions, monitors, and dependencies inherit from `VirtualClientComponent`. The runtime discovers components via reflection — no manual registration needed. + +**Lifecycle methods** (override these): + +1. `IsSupported()` — Check platform support (optional; also driven by `[SupportedPlatforms]` attribute) +2. `InitializeAsync(EventContext, CancellationToken)` — Download packages, set up state +3. `Validate()` — Verify parameters/preconditions (optional) +4. `ExecuteAsync(EventContext, CancellationToken)` — Run the workload, capture metrics +5. `CleanupAsync(EventContext, CancellationToken)` — Tear down resources (optional) + +### Execution Profiles + +Profiles are JSON files in `src/VirtualClient/VirtualClient.Main/profiles/` with three sections: + +- **Dependencies** — Run first; install packages/tools from blob storage +- **Actions** — Workload executors to run +- **Monitors** — Background system monitors + +Parameters support JPath references (`"$.Parameters.ProfilingEnabled"`) and environment variable substitution (`"{Environment:VAR_NAME}"`). + +Profile naming convention: `PERF--.json` (e.g., `PERF-CPU-COREMARK.json`, `PERF-IO-FIO.json`). + +### Client/Server Workloads + +Some workloads (e.g., HammerDB, network benchmarks) use a multi-VM client/server topology. This requires a `layout.json` file specifying IP addresses and roles: + +```json +{ + "clients": [ + { "name": "client-vm", "role": "Client", "privateIPAddress": "10.1.0.11" }, + { "name": "server-vm", "role": "Server", "privateIPAddress": "10.1.0.18" } + ] +} +``` + +Pass `--layout-path=/path/to/layout.json` when running VC on each VM. + +## Key Conventions + +### Implementing a New Workload + +1. Create a class in `VirtualClient.Actions` inheriting `VirtualClientComponent` +2. Add `[SupportedPlatforms("linux-x64,win-x64")]` attribute +3. Expose profile parameters as properties using `this.Parameters.GetValue(nameof(Property))` +4. Execute workloads via `this.ExecuteCommandAsync(exe, args, workingDir, telemetryContext, cancellationToken)` +5. Parse output into `IList` and log via `this.Logger.LogMetrics(...)` +6. Create a matching profile JSON in `VirtualClient.Main/profiles/` +7. Write unit tests in a matching `VirtualClient.Actions.UnitTests//` directory + +### Test Patterns + +- **Framework**: NUnit 3 + Moq + AutoFixture +- **Base class**: Test fixtures inherit from `MockFixture` (provides `IFileSystem`, `IPackageManager`, `ProcessManager`, `ISystemManagement`, etc. pre-mocked) +- **Example output files**: Store in `src/VirtualClient/TestResources/` and read via `MockFixture.ReadFile(MockFixture.ExamplesDirectory, "WorkloadName", "example-output.txt")` +- **Process mocking**: Use `this.ProcessManager.OnCreateProcess = (cmd, args, wd) => { /* assert args */ return this.Process; };` +- **Platform testing**: Call `this.Setup(PlatformID.Unix)` or `this.Setup(PlatformID.Win32NT)` in test setup + +### Dependency Package Installation + +Workload binaries/scripts are packaged as zip files in Azure Blob Storage. In profiles, use: + +```json +{ + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallMyWorkloadPackage", + "BlobContainer": "packages", + "BlobName": "myworkload.1.0.0.zip", + "PackageName": "myworkload", + "Extract": true + } +} +``` + +### Telemetry and Logging + +All operations are wrapped with `EventContext` for correlation. Use: + +- `this.Logger.LogMessage("Component.Operation", LogLevel.Information, telemetryContext)` for traces +- `this.Logger.LogMetrics("ToolName", metricName, value, unit, categorization, telemetryContext)` for workload results + +### Versioning + +The repo uses semantic versioning from the `VERSION` file at repo root (currently `3.0.5`). Override with `VCBuildVersion` environment variable. Central package management is enforced — all NuGet versions are in `Directory.Packages.props`. + +## Development Workflow + +### Fixing Source Code vs. Script Issues + +Issues will either require a **source code change** (C# in the VirtualClient solution) or a **script/package change** (workload scripts in blob storage). + +**For source code changes**: Edit, build, test, then deploy to VMs for validation. + +**For script/package changes**: Compress the updated files and upload to the VC packages blob store (`virtualclientinternal` storage account, `packages` container). + +- **Direct endpoint**: `https://virtualclientinternal.blob.core.windows.net/packages` +- **Azure Portal**: [packages container](https://ms.portal.azure.com/#view/Microsoft_Azure_Storage/ContainerMenuBlade/~/overview/storageAccountId/%2Fsubscriptions%2F94f4f5c5-3526-4f0d-83e5-2e7946a41b75%2FresourceGroups%2Fvirtualclient%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2Fvirtualclientinternal/path/packages/etag/%220x8DB982C9A0ACF93%22/defaultEncryptionScope/%24account-encryption-key/denyEncryptionScopeOverride~/false/defaultId//publicAccessVal/None) The package version must match the version referenced in the VC profile's `DependencyPackageInstallation` `BlobName`. If major script changes are needed from the previous commit, consult on incrementing the package version. + +### Testing on VMs + +After local unit/functional tests pass, validate on Azure VMs using the scripts in `~/OneDrive - Microsoft/Documents/create-vc-vms/`: + +```powershell +Import-Module -Name "C:\Users\evellanoweth\OneDrive - Microsoft\Documents\create-vc-vms\newVCvm.psm1" -Force + +# Single-VM workflow +New-VC-VM -vmName "my-test" -alias "evellanoweth" -vmSize "Standard_D2s_v5" +Build-VC # clean + build + package +Copy-Local-Item -vmName "my-test" -alias "evellanoweth" -itemPath "$vcPath\out\packages\VirtualClient.linux-x64.3.0.5.nupkg" +Extract-VC -vmName "my-test" -alias "evellanoweth" +Run-VC -vmName "my-test" -alias "evellanoweth" -vcArguments "--profile=PERF-WORKLOAD.json --packages= --verbose" + +# Client/Server workflow — create both VMs, copy VC + layout.json to each, run with --layout-path +``` + +Key VC runtime flags: `--profile`, `--packages` (blob store URL with managed identity), `--event-hub`, `--layout-path`, `--logger=csv`, `--verbose`, `--debug`, `-i=`. + +### Debugging Failures with Kusto + +When a VM run fails, check metrics and traces in Azure Data Explorer: + +- **Cluster**: `azurecrcworkloads.westus2.kusto.windows.net` +- **Metrics**: `WorkloadPerformance.Metrics_Dev01` — one row per metric data point (latency, throughput, IOPS) +- **Traces**: `WorkloadDiagnostics.Traces_Dev01` — diagnostic logs, errors, stack traces + +```kql +// Get error traces for a run +Traces_Dev01 +| where Timestamp > ago(1d) +| where SeverityLevel >= 3 +| where ProfileName == "PERF-MY-WORKLOAD" +| order by Timestamp asc + +// Check if workload produced metrics +Metrics_Dev01 +| where Timestamp > ago(1d) +| where MetricName !in ("Succeeded", "Failed") +| summarize count() by ToolName, MetricName +``` + +For PPE/production environments, swap the `_Dev01` suffix to `_PPE01`. The Juno cluster (`azurecrc.westus2.kusto.windows.net`) has experiment scheduling and failure data in `JunoIngestion` and `JunoStaging` databases. diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs index ae09915ce2..f7ae18a06d 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs @@ -276,7 +276,7 @@ public async Task SysbenchConfigurationProperlyExecutesTPCCPreparation() string[] expectedCommands = { - $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark TPCC --threadCount 8 --tableCount 10 --warehouses 0 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\"" + $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark TPCC --threadCount 8 --tableCount 10 --warehouses 1 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\"" }; int commandNumber = 0; @@ -332,7 +332,7 @@ public async Task SysbenchConfigurationProperlyExecutesTPCCConfigurablePreparati string[] expectedCommands = { - $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 999 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\"" + $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 1000 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\"" }; int commandNumber = 0; @@ -446,7 +446,7 @@ public async Task SysbenchConfigurationProperlyExecutesPostgreSQLTPCCConfigurabl string[] expectedCommands = { - $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 999 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\"" + $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 1000 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\"" }; int commandNumber = 0; diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs index 530dbb5d7f..edbf1a5868 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs @@ -345,7 +345,7 @@ private async Task ConfigureCreateHammerDBFile(EventContext telemetryContext, Ca } } - private async Task GenerateCommandLineArguments(CancellationToken cancellationToken) + private Task GenerateCommandLineArguments(CancellationToken cancellationToken) { string arguments = $"{this.PlatformSpecifics.Combine(this.HammerDBPackagePath, "configure-workload-generator.py")} --workload {this.Workload} --sqlServer {this.SQLServer} --port {this.Port}" + $" --virtualUsers {this.VirtualUsers} --password {this.SuperUserPassword} --dbName {this.DatabaseName} --hostIPAddress {this.ServerIpAddress}"; @@ -368,6 +368,8 @@ private async Task GenerateCommandLineArguments(CancellationToken cancellationTo } this.HammerDBScenarioArguments = arguments; + + return Task.CompletedTask; } private static Task OpenFirewallPortsAsync(int port, IFirewallManager firewallManager, CancellationToken cancellationToken) diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs index da48ffc57a..2862b60337 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs @@ -179,7 +179,7 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - this.sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments(SysbenchMode.Run); + this.sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments(); this.sysbenchExecutionArguments = $"{this.sysbenchLoggingArguments} --workload {this.Workload} --hostIpAddress {this.ServerIpAddress} --durationSecs {this.Duration.TotalSeconds} --password {this.SuperUserPassword}"; string script = $"{this.SysbenchPackagePath}/run-workload.py "; diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/Components/ExecuteCommandTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/Components/ExecuteCommandTests.cs new file mode 100644 index 0000000000..e0b936a0c4 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/Components/ExecuteCommandTests.cs @@ -0,0 +1,841 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient +{ + using System; + using System.Collections.Generic; + using System.Runtime.InteropServices; + using System.Threading; + using System.Threading.Tasks; + using Moq; + using NUnit.Framework; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + internal class ExecuteCommandTests + { + private MockFixture mockFixture; + + public void SetupDefaults(PlatformID platform, Architecture architecture = Architecture.X64) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(platform, architecture); + this.mockFixture.Parameters[nameof(ExecuteCommand.Command)] = "anycommand"; + } + + [Test] + [TestCase("anycommand", "anycommand", null)] + [TestCase("anycommand ", "anycommand", null)] + [TestCase("anycommand --argument=value", "anycommand", "--argument=value")] + [TestCase("/home/user/anycommand", "/home/user/anycommand", null)] + [TestCase("/home/user/anycommand --argument=value --argument2 value2", "/home/user/anycommand", "--argument=value --argument2 value2")] + [TestCase("\"/home/user/dir with space/anycommand\" --argument=value --argument2 value2", "\"/home/user/dir with space/anycommand\"", "--argument=value --argument2 value2")] + [TestCase("sudo anycommand", "sudo", "anycommand")] + [TestCase("sudo /home/user/anycommand", "sudo", "/home/user/anycommand")] + [TestCase("sudo /home/user/anycommand --argument=value --argument2 value2", "sudo", "/home/user/anycommand --argument=value --argument2 value2")] + [TestCase("sudo \"/home/user/dir with space/anycommand\" --argument=value --argument2 value2", "sudo", "\"/home/user/dir with space/anycommand\" --argument=value --argument2 value2")] + public async Task ExecuteCommandExecutesTheExpectedCommandOnUnixSystems(string fullCommand, string expectedCommand, string expectedCommandArguments) + { + this.SetupDefaults(PlatformID.Unix); + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(process.FullCommand(), $"{expectedCommand} {expectedCommandArguments}".Trim()); + }; + + await command.ExecuteAsync(CancellationToken.None); + } + } + + [Test] + [TestCase("anycommand.exe", "anycommand.exe", null)] + [TestCase("anycommand.exe ", "anycommand.exe", null)] + [TestCase("anycommand.exe --argument=value --argument2 value2", "anycommand.exe", "--argument=value --argument2 value2")] + [TestCase("C:\\Users\\User\\anycommand.exe", "C:\\Users\\User\\anycommand.exe", null)] + [TestCase("C:\\Users\\User\\anycommand.exe --argument=value --argument2 value2", "C:\\Users\\User\\anycommand.exe", "--argument=value --argument2 value2")] + [TestCase("\"C:\\Users\\User\\Dir With Space\\anycommand.exe\" --argument=value --argument2 value2", "\"C:\\Users\\User\\Dir With Space\\anycommand.exe\"", "--argument=value --argument2 value2")] + public async Task ExecuteCommandExecutesTheExpectedCommandOnWindowsSystems(string fullCommand, string expectedCommand, string expectedCommandArguments) + { + this.SetupDefaults(PlatformID.Win32NT); + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(process.FullCommand(), $"{expectedCommand} {expectedCommandArguments}".Trim()); + }; + + await command.ExecuteAsync(CancellationToken.None); + } + } + + [Test] + [TestCase("bash -c \"/home/user/anycommand&&/home/user/anyothercommand\"", "bash -c \"/home/user/anycommand&&/home/user/anyothercommand\"")] + [TestCase("bash -c \"/home/user/anycommand --argument=value&&/home/user/anyothercommand --argument2=value2\"", "bash -c \"/home/user/anycommand --argument=value&&/home/user/anyothercommand --argument2=value2\"")] + [TestCase("sudo bash -c \"/home/user/anycommand&&/home/user/anyothercommand\"", "sudo bash -c \"/home/user/anycommand&&/home/user/anyothercommand\"")] + [TestCase("sudo bash -c \"/home/user/anycommand --argument=value&&sudo /home/user/anyothercommand --argument2=value2\"", "sudo bash -c \"/home/user/anycommand --argument=value&&sudo /home/user/anyothercommand --argument2=value2\"")] + [TestCase("bash -c \"/home/user/anycommand --log-dir='home/user/log dir'&&/home/user/anyothercommand --log-dir='home/user/log dir'\"", "bash -c \"/home/user/anycommand --log-dir='home/user/log dir'&&/home/user/anyothercommand --log-dir='home/user/log dir'\"")] + [TestCase("bash -c '/home/user/anycommand --log-dir=\"home/user/log dir\"&&/home/user/anyothercommand --log-dir=\"home/user/log dir\"'", "bash -c '/home/user/anycommand --log-dir=\"home/user/log dir\"&&/home/user/anyothercommand --log-dir=\"home/user/log dir\"'")] + public async Task ExecuteCommandSupportsCommandChainingOnUnixSystems(string fullCommand, string expectedCommandExecuted) + { + this.SetupDefaults(PlatformID.Unix); + bool confirmed = false; + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommandExecuted, process.FullCommand()); + confirmed = true; + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(confirmed); + } + } + + [Test] + [TestCase("/home/user/anycommand&&/home/user/anyothercommand", "/home/user/anycommand;/home/user/anyothercommand")] + [TestCase("/home/user/anycommand --argument=value&&/home/user/anyothercommand --argument2=value2", "/home/user/anycommand --argument=value;/home/user/anyothercommand --argument2=value2")] + [TestCase("sudo anycommand&&anyothercommand", "sudo anycommand;sudo anyothercommand")] + [TestCase("sudo /home/user/anycommand&&/home/user/anyothercommand", "sudo /home/user/anycommand;sudo /home/user/anyothercommand")] + [TestCase("sudo /home/user/anycommand --argument=value&&/home/user/anyothercommand --argument2=value2", "sudo /home/user/anycommand --argument=value;sudo /home/user/anyothercommand --argument2=value2")] + public async Task ExecuteCommandSplitsAndExecutesChainedCommandsSequentiallyOnUnixSystems(string fullCommand, string expectedCommandExecuted) + { + this.SetupDefaults(PlatformID.Unix); + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + List expectedCommands = new List(expectedCommandExecuted.Split(';')); + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + expectedCommands.Remove(process.FullCommand()); + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsEmpty(expectedCommands); + } + } + + [Test] + [TestCase( + "sudo dmesg && sudo lsblk && sudo mount && sudo df -h && sudo find /sys -name scheduler -print", + "sudo dmesg;sudo lsblk;sudo mount;sudo df -h;sudo find /sys -name scheduler -print")] + public async Task ExecuteCommandSplitsChainedCommandsWithWhitespace(string fullCommand, string expectedCommandExecuted) + { + // Bug Scenario: + // Spaces (whitespace) in the commands due to the chaining SHOULD NOT cause + // parsing issues. + + this.SetupDefaults(PlatformID.Unix); + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + List expectedCommands = new List(expectedCommandExecuted.Split(';')); + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + expectedCommands.Remove(process.FullCommand()); + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsEmpty(expectedCommands); + } + } + + [Test] + [TestCase( + "git checkout 1.4.0&&autoreconf -ivf&&bash -c './configure'&&make", + "git checkout 1.4.0;autoreconf -ivf;bash -c './configure';make")] + public async Task ExecuteCommandSplitsChainedCommandsMatchingProfileUsage(string fullCommand, string expectedCommandExecuted) + { + // Scenario: Real profile usage from PERF-REDIS, PERF-MEMCACHED, GET-STARTED-REDIS. + // Commands chained with && should be split and executed as separate processes. + + this.SetupDefaults(PlatformID.Unix); + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + List expectedCommands = new List(expectedCommandExecuted.Split(';')); + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + expectedCommands.Remove(process.FullCommand()); + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsEmpty(expectedCommands); + } + } + + [Test] + [TestCase( + "dos2unix install-packages.sh&&dos2unix install-python.sh&&dos2unix install-pwsh.sh&&dos2unix install-docker.sh", + "dos2unix install-packages.sh;dos2unix install-python.sh;dos2unix install-pwsh.sh;dos2unix install-docker.sh")] + public async Task ExecuteCommandSplitsChainedCommandsWithRelativePaths(string fullCommand, string expectedCommandExecuted) + { + // Bug Scenario: + // Relative paths used to reference scripts in a specific working directory using a globally installed + // Linux toolset (e.g. dos2unix) should be left as-is. However, when a working directory is defined, that directory should + // be added to the PATH environment variable. + + this.SetupDefaults(PlatformID.Unix, Architecture.Arm64); + + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath/Platform:system_setup}"; + + this.mockFixture.PackageManager.OnGetPackage("system_setup") + .ReturnsAsync(new DependencyPath("system_setup", "/microsoft-labs/VirtualClient/content/linux-arm64/packages/system_setup.1.0.0")); + + string expectedWorkingDirectory = "/microsoft-labs/VirtualClient/content/linux-arm64/packages/system_setup.1.0.0/linux-arm64"; + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + List expectedCommands = new List(expectedCommandExecuted.Split(';')); + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + expectedCommands.Remove(process.FullCommand()); + + // Expect: + // The working directory of the process should be set. + Assert.AreEqual(expectedWorkingDirectory, process.StartInfo.WorkingDirectory); + + // Expect: + // The working directory should be added to the PATH environment variable. + Assert.IsTrue(this.mockFixture.PlatformSpecifics.EnvironmentVariables[EnvironmentVariable.PATH].Contains(expectedWorkingDirectory)); + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsEmpty(expectedCommands); + } + } + + [Test] + [TestCase("C:\\\\Users\\User\\anycommand&&C:\\\\home\\user\\anyothercommand", "C:\\\\Users\\User\\anycommand;C:\\\\home\\user\\anyothercommand")] + [TestCase("C:\\\\Users\\User\\anycommand --argument=1&&C:\\\\home\\user\\anyothercommand --argument=2", "C:\\\\Users\\User\\anycommand --argument=1;C:\\\\home\\user\\anyothercommand --argument=2")] + public async Task ExecuteCommandSplitsAndExecutesChainedCommandsSequentiallyOnWindowsSystems(string fullCommand, string expectedCommandExecuted) + { + this.SetupDefaults(PlatformID.Win32NT); + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + List expectedCommands = new List(expectedCommandExecuted.Split(';')); + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + expectedCommands.Remove(process.FullCommand()); + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsEmpty(expectedCommands); + } + } + + [Test] + [TestCase("/home/user/anycommand&&/home/user/anyothercommand", "bash -c \"/home/user/anycommand&&/home/user/anyothercommand\"")] + [TestCase("/home/user/anycommand --argument=value&&/home/user/anyothercommand --argument2=value2", "bash -c \"/home/user/anycommand --argument=value&&/home/user/anyothercommand --argument2=value2\"")] + [TestCase("sudo /home/user/anycommand&&/home/user/anyothercommand", "bash -c \"sudo /home/user/anycommand&&/home/user/anyothercommand\"")] + [TestCase("sudo /home/user/anycommand --argument=value&&sudo /home/user/anyothercommand --argument2=value2", "bash -c \"sudo /home/user/anycommand --argument=value&&sudo /home/user/anyothercommand --argument2=value2\"")] + public async Task ExecuteCommandSupportsCommandChainingOnUnixSystems_With_UseShell(string fullCommand, string expectedCommandExecuted) + { + this.SetupDefaults(PlatformID.Unix); + this.mockFixture.Parameters[nameof(ExecuteCommand.UseShell)] = true; + bool confirmed = false; + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommandExecuted, process.FullCommand()); + confirmed = true; + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(confirmed); + } + } + + [Test] + [TestCase( + "sudo dmesg && sudo lsblk && sudo mount && sudo df -h && sudo find /sys -name scheduler -print", + "bash -c \"sudo dmesg && sudo lsblk && sudo mount && sudo df -h && sudo find /sys -name scheduler -print\"")] + public async Task ExecuteCommandSupportsCommandChainingOnUnixSystems_Bug_1(string fullCommand, string expectedCommandExecuted) + { + // Bug Scenario: + // Spaces (whitespace) in the commands due to the chaining SHOULD NOT cause + // parsing issues. + // + // e.g. + // "sudo dmesg && sudo lsblk " resulting in the command being identified as "sudo o lsblk" + + this.SetupDefaults(PlatformID.Unix); + this.mockFixture.Parameters[nameof(ExecuteCommand.UseShell)] = true; + bool confirmed = false; + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommandExecuted, process.FullCommand()); + confirmed = true; + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(confirmed); + } + } + + [Test] + [TestCase( + "dos2unix install-packages.sh&&dos2unix install-python.sh&&dos2unix install-pwsh.sh&&dos2unix install-docker.sh", + "bash -c \"dos2unix install-packages.sh&&dos2unix install-python.sh&&dos2unix install-pwsh.sh&&dos2unix install-docker.sh\"")] + public async Task ExecuteCommandSupportsCommandChainingOnUnixSystems_Bug_2(string fullCommand, string expectedCommandExecuted) + { + // Bug Scenario: + // Relative paths used to reference scripts in a specific working directory using a globally installed + // Linux toolset (e.g. dos2unix) should be left as-is. However, when a working directory is defined, that directory should + // be added to the PATH environment variable. + // + // e.g. + // (given working directory /microsoft-labs/VirtualClient/content/linux-arm64/packages/system_setup.1.0.0/linux-arm64) + // + // Should Be: + // the working + + this.SetupDefaults(PlatformID.Unix, Architecture.Arm64); + this.mockFixture.Parameters[nameof(ExecuteCommand.UseShell)] = true; + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath/Platform:system_setup}"; + bool confirmed = false; + + this.mockFixture.PackageManager.OnGetPackage("system_setup") + .ReturnsAsync(new DependencyPath("system_setup", "/microsoft-labs/VirtualClient/content/linux-arm64/packages/system_setup.1.0.0")); + + string expectedWorkingDirectory = "/microsoft-labs/VirtualClient/content/linux-arm64/packages/system_setup.1.0.0/linux-arm64"; + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommandExecuted, process.FullCommand()); + + // Expect: + // The working directory of the process should be set. + Assert.AreEqual(expectedWorkingDirectory, process.StartInfo.WorkingDirectory); + + // Expect: + // The working directory should be added to the PATH environment variable. + Assert.IsTrue(this.mockFixture.PlatformSpecifics.EnvironmentVariables[EnvironmentVariable.PATH].Contains(expectedWorkingDirectory)); + confirmed = true; + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(confirmed); + } + } + + [Test] + [TestCase("C:\\\\Users\\User\\anycommand&&C:\\\\home\\user\\anyothercommand", "cmd /C \"C:\\\\Users\\User\\anycommand&&C:\\\\home\\user\\anyothercommand\"")] + [TestCase("C:\\\\Users\\User\\anycommand --argument=1&&C:\\\\home\\user\\anyothercommand --argument=2\"", "cmd /C \"C:\\\\Users\\User\\anycommand --argument=1&&C:\\\\home\\user\\anyothercommand --argument=2\"")] + public async Task ExecuteCommandSupportsCommandChainingOnWindowsSystems(string fullCommand, string expectedCommandExecuted) + { + this.SetupDefaults(PlatformID.Win32NT); + this.mockFixture.Parameters[nameof(ExecuteCommand.UseShell)] = true; + bool confirmed = false; + + using (TestExecuteCommand command = new TestExecuteCommand(this.mockFixture)) + { + command.Parameters[nameof(ExecuteCommand.Command)] = fullCommand; + List expectedCommands = new List(expectedCommandExecuted.Split(';')); + + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommandExecuted, process.FullCommand()); + confirmed = true; + }; + + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(confirmed); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task ExecuteCommandExecuteCommandsWhenThePlatformMatchesTheOnesDefinedInTheParameters_UnixSystems_SinglePlatformSpecified(PlatformID platform, Architecture architecture) + { + this.SetupDefaults(platform, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = PlatformSpecifics.GetPlatformArchitectureName(platform, architecture); + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(commandExecuted); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task ExecuteCommandExecuteCommandsWhenThePlatformMatchesTheOnesDefinedInTheParameters_UnixSystems_MultiplePlatformsSpecified(PlatformID platform, Architecture architecture) + { + // Setup running on a Unix system but the supported platforms are + // Windows systems. + this.SetupDefaults(platform, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Unix, Architecture.X64)}," + + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Unix, Architecture.Arm64)}"; + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(commandExecuted); + } + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public async Task ExecuteCommandExecuteCommandsWhenThePlatformMatchesTheOnesDefinedInTheParameters_WindowsSystems_SinglePlatformSpecified(PlatformID platform, Architecture architecture) + { + this.SetupDefaults(platform, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = PlatformSpecifics.GetPlatformArchitectureName(platform, architecture); + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(commandExecuted); + } + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public async Task ExecuteCommandExecuteCommandsWhenThePlatformMatchesTheOnesDefinedInTheParameters_WindowsSystems_MultiplePlatformsSpecified(PlatformID platform, Architecture architecture) + { + // Setup running on a Unix system but the supported platforms are + // Windows systems. + this.SetupDefaults(platform, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Win32NT, Architecture.X64)}," + + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Win32NT, Architecture.Arm64)}"; + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(commandExecuted); + } + } + + [Test] + [TestCase(Architecture.X64)] + [TestCase(Architecture.Arm64)] + public async Task ExecuteCommandDoesNotExecuteCommandsUnlessThePlatformMatchesTheOnesDefinedInTheParameters_UnixSystems_SinglePlatformSpecified(Architecture architecture) + { + // Setup running on a Unix system but the supported platforms are + // Windows systems. + this.SetupDefaults(PlatformID.Unix, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Win32NT, architecture); + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsFalse(commandExecuted); + } + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task ExecuteCommandDoesNotExecuteCommandsUnlessThePlatformMatchesTheOnesDefinedInTheParameters_UnixSystems_MultiplePlatformSpecified(PlatformID platform, Architecture architecture) + { + // Setup running on a Unix system but the supported platforms are + // Windows systems. + this.SetupDefaults(platform, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Win32NT, Architecture.X64)}," + + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Win32NT, Architecture.Arm64)}"; + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsFalse(commandExecuted); + } + } + + [Test] + [TestCase(Architecture.X64)] + [TestCase(Architecture.Arm64)] + public async Task ExecuteCommandDoesNotExecuteCommandsUnlessThePlatformMatchesTheOnesDefinedInTheParameters_WindowsSystems_SinglePlatformSpecified(Architecture architecture) + { + // Setup running on a Unix system but the supported platforms are + // Windows systems. + this.SetupDefaults(PlatformID.Win32NT, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Unix, architecture); + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsFalse(commandExecuted); + } + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public async Task ExecuteCommandDoesNotExecuteCommandsUnlessThePlatformMatchesTheOnesDefinedInTheParameters_WindowsSystems_MultiplePlatformSpecified(PlatformID platform, Architecture architecture) + { + // Setup running on a Unix system but the supported platforms are + // Windows systems. + this.SetupDefaults(platform, architecture); + + // e.g. + // linux-x64, win-arm64 + this.mockFixture.Parameters[nameof(ExecuteCommand.SupportedPlatforms)] = + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Unix, Architecture.X64)}," + + $"{PlatformSpecifics.GetPlatformArchitectureName(PlatformID.Unix, Architecture.Arm64)}"; + + bool commandExecuted = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => commandExecuted = true; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + // We SHOULD NOT execute on the system when the platform/architecture does not match. + await command.ExecuteAsync(CancellationToken.None); + Assert.IsFalse(commandExecuted); + } + } + + [Test] + public async Task ExecuteCommandResolvesPackagePathExpressionsOnInitializationInTheComponentParametersOnWindowsSystems() + { + this.SetupDefaults(PlatformID.Win32NT); + string packagePath = this.mockFixture.GetPackagePath("anypackage"); + + // The package MUST exist on the system. + this.mockFixture.PackageManager.OnGetPackage("anypackage").ReturnsAsync(new DependencyPath("anypackage", packagePath)); + + // The component uses the {PackagePath} referencing expression in both command and + // working directory parameters. + this.mockFixture.Parameters[nameof(ExecuteCommand.Command)] = "{PackagePath:anypackage}\\build.exe&&{PackagePath:anypackage}\\build.exe install"; + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath:anypackage}"; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + await command.InitializeAsync(EventContext.None, CancellationToken.None); + + Assert.AreEqual($"{packagePath}\\build.exe&&{packagePath}\\build.exe install", command.Parameters[nameof(ExecuteCommand.Command)]); + Assert.AreEqual(packagePath, command.Parameters[nameof(ExecuteCommand.WorkingDirectory)]); + } + } + + [Test] + public async Task ExecuteCommandResolvesPackagePathExpressionsOnInitializationInTheComponentParametersOnUnixSystems() + { + this.SetupDefaults(PlatformID.Unix); + string packagePath = this.mockFixture.GetPackagePath("anypackage"); + + // The package MUST exist on the system. + this.mockFixture.PackageManager.OnGetPackage("anypackage").ReturnsAsync(new DependencyPath("anypackage", packagePath)); + + // The component uses the {PackagePath} referencing expression in both command and + // working directory parameters. + this.mockFixture.Parameters[nameof(ExecuteCommand.Command)] = "{PackagePath:anypackage}/configure&&{PackagePath:anypackage}/make"; + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath:anypackage}"; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + await command.InitializeAsync(EventContext.None, CancellationToken.None); + + Assert.AreEqual($"{packagePath}/configure&&{packagePath}/make", command.Parameters[nameof(ExecuteCommand.Command)]); + Assert.AreEqual(packagePath, command.Parameters[nameof(ExecuteCommand.WorkingDirectory)]); + } + } + + [Test] + [Platform("Win")] + public async Task ExecuteCommandTheResolvedPackagePathExpressionsWhenExecutingCommandsOnWindowsSystems() + { + this.SetupDefaults(PlatformID.Win32NT); + string packagePath = this.mockFixture.GetPackagePath("anypackage"); + + // The package MUST exist on the system. + this.mockFixture.PackageManager.OnGetPackage("anypackage").ReturnsAsync(new DependencyPath("anypackage", packagePath)); + + // The component uses the {PackagePath} referencing expression in both command and + // working directory parameters. + this.mockFixture.Parameters[nameof(ExecuteCommand.UseShell)] = true; + this.mockFixture.Parameters[nameof(ExecuteCommand.Command)] = "{PackagePath:anypackage}\\build.exe&&{PackagePath:anypackage}\\build.exe install"; + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath:anypackage}"; + + string expectedCommand = $"cmd /C \"{packagePath}\\build.exe&&{packagePath}\\build.exe install\"";; + + bool confirmed = false; + using (var command = new TestExecuteCommand(this.mockFixture)) + { + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommand, process.FullCommand()); + Assert.AreEqual(packagePath, process.StartInfo.WorkingDirectory); + confirmed = true; + }; + + await command.ExecuteAsync(CancellationToken.None); + + Assert.IsTrue(confirmed); + } + } + + [Test] + public async Task ExecuteCommandTheResolvedPackagePathExpressionsWhenExecutingCommandsOnUnixSystems() + { + this.SetupDefaults(PlatformID.Unix); + string packagePath = this.mockFixture.GetPackagePath("anypackage"); + + // The package MUST exist on the system. + this.mockFixture.PackageManager.OnGetPackage("anypackage").ReturnsAsync(new DependencyPath("anypackage", packagePath)); + + // The component uses the {PackagePath} referencing expression in both command and + // working directory parameters. + this.mockFixture.Parameters[nameof(ExecuteCommand.UseShell)] = true; + this.mockFixture.Parameters[nameof(ExecuteCommand.Command)] = "{PackagePath:anypackage}/configure&&{PackagePath:anypackage}/make"; + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath:anypackage}"; + + string expectedCommand = $"bash -c \"{packagePath}/configure&&{packagePath}/make\""; + bool confirmed = false; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommand, process.FullCommand()); + Assert.AreEqual(packagePath, process.StartInfo.WorkingDirectory); + confirmed = true; + }; + + await command.ExecuteAsync(CancellationToken.None); + + Assert.IsTrue(confirmed); + } + } + + [Test] + public async Task ExecuteCommandResolvesPlatformSpecificPackagePathExpressionsOnInitializationInTheComponentParametersOnWindowsSystems() + { + this.SetupDefaults(PlatformID.Win32NT); + string packagePath = this.mockFixture.GetPackagePath("anypackage"); + string platformSpecificPath = this.mockFixture.Combine(packagePath, "win-x64"); + + // The package MUST exist on the system. + this.mockFixture.PackageManager.OnGetPackage("anypackage").ReturnsAsync(new DependencyPath("anypackage", packagePath)); + + // The component uses the {PackagePath/Platform} referencing expression in both command and + // working directory parameters. + this.mockFixture.Parameters[nameof(ExecuteCommand.Command)] = "{PackagePath/Platform:anypackage}\\build.exe&&{PackagePath/Platform:anypackage}\\build.exe install"; + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath/Platform:anypackage}"; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + await command.InitializeAsync(EventContext.None, CancellationToken.None); + + Assert.AreEqual($"{platformSpecificPath}\\build.exe&&{platformSpecificPath}\\build.exe install", command.Parameters[nameof(ExecuteCommand.Command)]); + Assert.AreEqual(platformSpecificPath, command.Parameters[nameof(ExecuteCommand.WorkingDirectory)]); + } + } + + [Test] + public async Task ExecuteCommandResolvesPlatformSpecificPackagePathExpressionsOnInitializationInTheComponentParametersOnUnixSystems() + { + this.SetupDefaults(PlatformID.Unix); + string packagePath = this.mockFixture.GetPackagePath("anypackage"); + string platformSpecificPath = this.mockFixture.Combine(packagePath, "linux-x64"); + + // The package MUST exist on the system. + this.mockFixture.PackageManager.OnGetPackage("anypackage").ReturnsAsync(new DependencyPath("anypackage", packagePath)); + + // The component uses the {PackagePath} referencing expression in both command and + // working directory parameters. + this.mockFixture.Parameters[nameof(ExecuteCommand.Command)] = "{PackagePath/Platform:anypackage}/configure&&{PackagePath/Platform:anypackage}/make"; + this.mockFixture.Parameters[nameof(ExecuteCommand.WorkingDirectory)] = "{PackagePath/Platform:anypackage}"; + + using (var command = new TestExecuteCommand(this.mockFixture)) + { + await command.InitializeAsync(EventContext.None, CancellationToken.None); + + Assert.AreEqual($"{platformSpecificPath}/configure&&{platformSpecificPath}/make", command.Parameters[nameof(ExecuteCommand.Command)]); + Assert.AreEqual(platformSpecificPath, command.Parameters[nameof(ExecuteCommand.WorkingDirectory)]); + } + } + + [Test] + public async Task ExecuteCommandHandlesDotNetAnomaliesWhenWorkingDirectoriesAreDefined_1() + { + this.SetupDefaults(PlatformID.Unix); + + string command = "anyscript.sh"; + string workingDirectory = "/home/user/scripts"; + string expectedCommand = command; + string expectedWorkingDirectory = workingDirectory; + + using (var component = new TestExecuteCommand(this.mockFixture)) + { + component.Parameters[nameof(TestExecuteCommand.Command)] = command; + component.Parameters[nameof(TestExecuteCommand.WorkingDirectory)] = workingDirectory; + + bool confirmed = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommand, process.FullCommand()); + Assert.AreEqual(expectedWorkingDirectory, process.StartInfo.WorkingDirectory); + + // The working directory is added to the process PATH environment variable + // to avoid issues with .NET implementations. + // + // Context: + // There appears to be an unfortunate implementation choice in .NET causing a Win32Exception similar to the following when + // referencing a binary and setting the working directory. + // + // System.ComponentModel.Win32Exception: + // 'An error occurred trying to start process 'Coreinfo64.exe' with working directory 'S:\microsoft\virtualclient\out\bin\Debug\AnyCPU\VirtualClient.Main\net9.0\packages\system_tools\win-x64'. + // The system cannot find the file specified. + // + // The .NET Process class does not reference the 'WorkingDirectory' when looking for the 'FileName' when UseShellExecute = false. The workaround + // for this is to add the working directory to the PATH environment variable. + Assert.IsTrue(component.PlatformSpecifics.GetEnvironmentVariable(EnvironmentVariable.PATH).Contains(workingDirectory)); + confirmed = true; + }; + + await component.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(confirmed); + } + } + + [Test] + public async Task ExecuteCommandMonitorDotNetAnomaliesWhenWorkingDirectoriesAreDefined_2() + { + this.SetupDefaults(PlatformID.Unix); + + string command = "\"anyscript.sh\""; + string workingDirectory = "/home/user/scripts"; + string expectedCommand = $"\"anyscript.sh\""; + string expectedWorkingDirectory = workingDirectory; + + using (var component = new TestExecuteCommand(this.mockFixture)) + { + component.Parameters[nameof(TestExecuteCommand.Command)] = command; + component.Parameters[nameof(TestExecuteCommand.WorkingDirectory)] = workingDirectory; + + bool confirmed = false; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + Assert.AreEqual(expectedCommand, process.FullCommand()); + Assert.AreEqual(expectedWorkingDirectory, process.StartInfo.WorkingDirectory); + + // The working directory is added to the process PATH environment variable + // to avoid issues with .NET implementations. + // + // Context: + // There appears to be an unfortunate implementation choice in .NET causing a Win32Exception similar to the following when + // referencing a binary and setting the working directory. + // + // System.ComponentModel.Win32Exception: + // 'An error occurred trying to start process 'Coreinfo64.exe' with working directory 'S:\microsoft\virtualclient\out\bin\Debug\AnyCPU\VirtualClient.Main\net9.0\packages\system_tools\win-x64'. + // The system cannot find the file specified. + // + // The .NET Process class does not reference the 'WorkingDirectory' when looking for the 'FileName' when UseShellExecute = false. The workaround + // for this is to add the working directory to the PATH environment variable. + Assert.IsTrue(component.PlatformSpecifics.GetEnvironmentVariable(EnvironmentVariable.PATH).Contains(workingDirectory)); + confirmed = true; + }; + + await component.ExecuteAsync(CancellationToken.None); + Assert.IsTrue(confirmed); + } + } + + private class TestExecuteCommand : ExecuteCommand + { + public TestExecuteCommand(MockFixture mockFixture) + : base(mockFixture?.Dependencies, mockFixture?.Parameters) + { + } + + public new Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return base.InitializeAsync(telemetryContext, cancellationToken); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Core/Components/ExecuteCommand.cs b/src/VirtualClient/VirtualClient.Core/Components/ExecuteCommand.cs new file mode 100644 index 0000000000..5e3e19d851 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Core/Components/ExecuteCommand.cs @@ -0,0 +1,317 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// Executes a command on the system with the working directory set to a + /// package installed. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] + public class ExecuteCommand : VirtualClientComponent + { + private static readonly char[] Quotes = new char[] { '"', '\'' }; + private ProcessManager processManager; + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component + /// A series of key value pairs that dictate runtime execution. + public ExecuteCommand(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.processManager = dependencies.GetService(); + + this.RetryPolicy = Policy.Handle().WaitAndRetryAsync( + this.MaxRetries, + (retries) => TimeSpan.FromSeconds(retries + 1)); + } + + /// + /// Parameter defines the command(s) to execute. Multiple commands should be delimited using + /// the '&&' characters which works on both Windows and Unix systems (e.g. ./configure&&make). + /// + public string Command + { + get + { + return this.Parameters.GetValue(nameof(this.Command)); + } + } + + /// + /// Provides a set of environment variables to add to the monitor process executions. + /// + public IDictionary EnvironmentVariables + { + get + { + IDictionary environmentVariables = null; + this.Parameters.TryGetValue(nameof(this.EnvironmentVariables), out IConvertible variables); + + if (variables != null) + { + environmentVariables = TextParsingExtensions.ParseDelimitedValues(variables?.ToString()); + } + + return environmentVariables; + } + } + + /// + /// Parameter defines the maximum number of retries on failures of the + /// command execution. Default = 0; + /// + public int MaxRetries + { + get + { + return this.Parameters.GetValue(nameof(this.MaxRetries), 0); + } + } + + /// + /// A policy that defines how the component will retry when it experiences transient issues. + /// + public IAsyncPolicy RetryPolicy { get; set; } + + /// + /// True/false to indicate that a shell (bash, cmd) should be used to execute the command. The user can also simply define the + /// shell that they want to use in the command line directly. This supports backwards compatibility scenarios. Default = false. + /// + public bool UseShell + { + get + { + return this.Parameters.GetValue(nameof(this.UseShell), false); + } + } + + /// + /// Parameter defines the working directory from which the command should be executed. When the + /// 'PackageName' parameter is defined, this parameter will take precedence. Otherwise, the directory + /// where the package is installed for the 'PackageName' parameter will be used as the working directory. + /// + public string WorkingDirectory + { + get + { + this.Parameters.TryGetValue(nameof(this.WorkingDirectory), out IConvertible workingDir); + return workingDir?.ToString(); + } + } + + /// + /// Execute the command(s) logic on the system. + /// + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + telemetryContext.AddContext("command", SensitiveData.ObscureSecrets(this.Command)); + telemetryContext.AddContext("workingDirectory", SensitiveData.ObscureSecrets(this.WorkingDirectory)); + telemetryContext.AddContext("platforms", string.Join(VirtualClientComponent.CommonDelimiters.First(), this.SupportedPlatforms)); + + if (!cancellationToken.IsCancellationRequested) + { + IEnumerable commandsToExecute = this.GetCommandsToExecute(); + + if (commandsToExecute?.Any() == true) + { + IDictionary environmentVariables = this.EnvironmentVariables; + foreach (string originalCommand in commandsToExecute) + { + if (!cancellationToken.IsCancellationRequested) + { + if (PlatformSpecifics.TryGetCommandParts(originalCommand, out string effectiveCommand, out string effectiveCommandArguments)) + { + await (this.RetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + // There appears to be an unfortunate implementation choice in .NET causing a Win32Exception similar to the following when + // referencing a binary and setting the working directory. + // + // System.ComponentModel.Win32Exception: + // 'An error occurred trying to start process 'Coreinfo64.exe' with working directory 'S:\microsoft\virtualclient\out\bin\Debug\AnyCPU\VirtualClient.Main\net9.0\packages\system_tools\win-x64'. + // The system cannot find the file specified. + // + // The .NET Process class does not reference the 'WorkingDirectory' when looking for the 'FileName' when UseShellExecute = false. The workaround + // for this is to add the working directory to the PATH environment variable. + string effectiveWorkingDirectory = this.WorkingDirectory; + if (!string.IsNullOrWhiteSpace(effectiveWorkingDirectory)) + { + this.PlatformSpecifics.SetEnvironmentVariable( + EnvironmentVariable.PATH, + effectiveWorkingDirectory, + EnvironmentVariableTarget.Process, + append: true); + } + + using (IProcessProxy process = this.processManager.CreateProcess(effectiveCommand, effectiveCommandArguments, effectiveWorkingDirectory)) + { + this.AddEnvironmentVariables(process, environmentVariables); + await process.StartAndWaitAsync(cancellationToken); + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, logFileName: this.LogFileName); + process.ThrowIfComponentOperationFailed(this.ComponentType); + } + } + }); + } + } + } + } + } + } + + /// + /// Initializes the component for execution. + /// + protected override Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.EvaluateParametersAsync(cancellationToken); + } + + /// + /// Returns true/false whether the component is supported on the system. + /// + protected override bool IsSupported() + { + bool isSupported = false; + if (base.IsSupported()) + { + // We execute only if the current platform/architecture matches those + // defined in the parameters. + if (!this.SupportedPlatforms.Any() || this.SupportedPlatforms.Contains(this.PlatformArchitectureName)) + { + isSupported = true; + } + } + + return isSupported; + } + + private void AddEnvironmentVariables(IProcessProxy process, IDictionary environmentVariables) + { + if (environmentVariables?.Any() == true) + { + foreach (var entry in environmentVariables) + { + string variableName = entry.Key.Trim(); + string variableValue = entry.Value?.ToString().Trim(); + + if (string.IsNullOrWhiteSpace(variableValue) && process.EnvironmentVariables.ContainsKey(variableName)) + { + process.EnvironmentVariables.Remove(variableName); + } + else if (variableValue.StartsWith("+")) + { + this.PlatformSpecifics.SetEnvironmentVariable(process, variableName, variableValue.Substring(1).Trim(), true); + } + else + { + this.PlatformSpecifics.SetEnvironmentVariable(process, variableName, variableValue); + } + } + } + } + + private IEnumerable GetCommandsToExecute() + { + if (this.UseShell) + { + // When using a shell (e.g. bash, cmd), the entire command is passed to the shell as a single + // process and the shell handles any chaining (e.g. &&). + string targetCommand = this.Command; + switch (this.Platform) + { + case PlatformID.Unix: + targetCommand = $"bash -c \"{targetCommand.Trim(ExecuteCommand.Quotes)}\""; + break; + + case PlatformID.Win32NT: + targetCommand = $"cmd /C \"{targetCommand.Trim(ExecuteCommand.Quotes)}\""; + break; + } + + return new List { targetCommand }; + } + + // When not using a shell, split on '&&' and execute each command sequentially as separate processes. + // The split is quote-aware: '&&' inside single or double quotes is not treated as a delimiter. + List commandsToExecute = new List(); + bool sudo = this.Command.StartsWith("sudo", StringComparison.OrdinalIgnoreCase); + + IEnumerable commands = ExecuteCommand.SplitOnChainOperator(this.Command); + + foreach (string fullCommand in commands) + { + if (sudo && !fullCommand.Contains("sudo", StringComparison.OrdinalIgnoreCase)) + { + commandsToExecute.Add($"sudo {fullCommand}"); + } + else + { + commandsToExecute.Add(fullCommand); + } + } + + return commandsToExecute; + } + + private static IEnumerable SplitOnChainOperator(string command) + { + List parts = new List(); + char? quoteChar = null; + int segmentStart = 0; + + for (int i = 0; i < command.Length; i++) + { + char c = command[i]; + + if (quoteChar != null) + { + if (c == quoteChar) + { + quoteChar = null; + } + } + else if (c == '"' || c == '\'') + { + quoteChar = c; + } + else if (c == '&' && i + 1 < command.Length && command[i + 1] == '&') + { + string segment = command.Substring(segmentStart, i - segmentStart).Trim(); + if (segment.Length > 0) + { + parts.Add(segment); + } + + i++; + segmentStart = i + 1; + } + } + + string lastSegment = command.Substring(segmentStart).Trim(); + if (lastSegment.Length > 0) + { + parts.Add(lastSegment); + } + + return parts; + } + } +} diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs index 3e53289d33..731f385443 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs @@ -58,7 +58,7 @@ public async Task MySQLConfigurationExecutesTheExpectedProcessForConfigureServer string[] expectedCommands = { - $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1/mysql;/home/user/mnt_dev_sdd1/mysql;/home/user/mnt_dev_sde1/mysql\"", + $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1/mysql\"", }; int commandNumber = 0; @@ -115,7 +115,7 @@ public async Task MySQLConfigurationExecutesTheExpectedProcessForConfigureServer string[] expectedCommands = { - $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1/mysql;/home/user/mnt_dev_sdd1/mysql;/home/user/mnt_dev_sde1/mysql\" " + + $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1/mysql\" " + $"--inMemory 8192", }; diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs index c4b431fa2f..b13ad639af 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs @@ -75,7 +75,7 @@ public async Task PostgreSQLServerConfigurationExecutesTheExpectedProcessForConf string[] expectedCommands = { - $"python3 {tempPackagePath}/configure-server.py --dbName hammerdbtest --serverIp 1.2.3.5 --password [A-Za-z0-9+/=]+ --port 5432 --inMemory [0-9]+", + $"python3 {tempPackagePath}/configure-server.py --dbName hammerdbtest --serverIp 1.2.3.5 --password [A-Za-z0-9+/=]+ --port 5432 --inMemory [0-9]+ --directory \\S+", }; int commandNumber = 0; diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/StripeDisksTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/StripeDisksTests.cs index 5558f26c5f..43790afa39 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/StripeDisksTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/StripeDisksTests.cs @@ -289,6 +289,30 @@ public async Task StripeDisksExecutesTheExpectedCommandOnWindows() Assert.IsTrue(confirmed); } + [Test] + public async Task StripeDisksSkipsWhenMountDirectoryAlreadyHasContent() + { + this.mockFixture.Parameters["DiskFilter"] = "OSDisk:false"; + + string expectedMountDir = $"/home/{Environment.UserName}/mnt_raid0"; + this.mockFixture.Directory.Setup(d => d.Exists(expectedMountDir)).Returns(true); + this.mockFixture.Directory.Setup(d => d.EnumerateFileSystemEntries(expectedMountDir)) + .Returns(new[] { $"{expectedMountDir}/lost+found" }); + + int commandsExecuted = 0; + this.mockFixture.ProcessManager.OnProcessCreated = (process) => + { + commandsExecuted++; + }; + + using (StripeDisks component = new StripeDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await component.ExecuteAsync(CancellationToken.None); + } + + Assert.AreEqual(0, commandsExecuted); + } + private void SetupSystemConfigPackage() { this.systemConfigPackage = new DependencyPath( diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 15726bd8f9..75fb3fd4f5 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -63,22 +63,6 @@ public bool SkipInitialize } } - /// - /// stripedisk mount point. - /// - public string StripeDiskMountPoint - { - get - { - if (this.Parameters.TryGetValue(nameof(this.StripeDiskMountPoint), out IConvertible stripediskmountpoint) && stripediskmountpoint != null) - { - return stripediskmountpoint.ToString(); - } - - return string.Empty; - } - } - /// /// Disk filter specified /// @@ -86,7 +70,6 @@ public string DiskFilter { get { - // and 256G return this.Parameters.GetValue(nameof(this.DiskFilter), "osdisk:false&sizegreaterthan:256g"); } } @@ -134,7 +117,6 @@ public bool InMemory /// protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) { - ProcessManager manager = this.SystemManager.ProcessManager; string stateId = $"{nameof(MySQLServerConfiguration)}-{this.Action}-action-success"; ConfigurationState configurationState = await this.stateManager.GetStateAsync(stateId, cancellationToken) .ConfigureAwait(false); @@ -178,7 +160,7 @@ private async Task ConfigureMySQLServerAsync(EventContext telemetryContext, Canc .FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); - string innoDbDirs = !string.IsNullOrEmpty(this.StripeDiskMountPoint) ? this.StripeDiskMountPoint : await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); + string innoDbDirs = await this.GetMySQLInnodbDirectoriesAsync(cancellationToken); string arguments = $"{this.packageDirectory}/configure.py --serverIp {serverIp} --innoDbDirs \"{innoDbDirs}\""; @@ -249,46 +231,93 @@ private async Task SetMySQLGlobalVariableAsync(EventContext telemetryContext, Ca private async Task GetMySQLInnodbDirectoriesAsync(CancellationToken cancellationToken) { - string diskPaths = string.Empty; + IEnumerable disks = await this.SystemManager.DiskManager.GetDisksAsync(cancellationToken) + .ConfigureAwait(false); + + IEnumerable filteredDisks = DiskFilters.FilterDisks(disks, this.DiskFilter, this.Platform); + + this.Logger.LogTraceMessage($"{this.TypeName}: Total disks discovered: {disks.Count()}. Disks after filtering ('{this.DiskFilter}'): {filteredDisks.Count()}."); + + // Search ALL disks for a raid0 mount point. After StripeDisks creates an LVM + // striped volume from the filtered physical disks, the mount point appears on + // the logical volume device, which is a separate disk entry from the originals. + string raidAccessPath = disks + .SelectMany(d => d.Volumes) + .SelectMany(v => v.AccessPaths) + .FirstOrDefault(p => p.Contains("raid0", StringComparison.OrdinalIgnoreCase)); - if (!cancellationToken.IsCancellationRequested) + // lshw may not report LVM logical volumes at all. Fall back to reading + // /proc/mounts which always lists every active mount point. + if (string.IsNullOrEmpty(raidAccessPath) && this.Platform != PlatformID.Win32NT) { - IEnumerable disks = await this.SystemManager.DiskManager.GetDisksAsync(cancellationToken) + try + { + string procMounts = await this.SystemManager.FileSystem.File.ReadAllTextAsync("/proc/mounts", cancellationToken) .ConfigureAwait(false); - if (disks?.Any() != true) + foreach (string line in procMounts.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + string[] parts = line.Split(' '); + if (parts.Length >= 2 && parts[1].Contains("raid0", StringComparison.OrdinalIgnoreCase)) + { + raidAccessPath = parts[1]; + this.Logger.LogTraceMessage($"{this.TypeName}: Found raid0 mount from /proc/mounts: '{raidAccessPath}'."); + break; + } + } + } + catch (Exception ex) { - throw new WorkloadException( - "Unexpected scenario. The disks defined for the system could not be properly enumerated.", - ErrorReason.WorkloadUnexpectedAnomaly); + this.Logger.LogTraceMessage($"{this.TypeName}: Could not read /proc/mounts: {ex.Message}"); } + } - IEnumerable disksToTest = DiskFilters.FilterDisks(disks, this.DiskFilter, this.Platform).ToList(); + string accessPath = raidAccessPath; - if (disksToTest?.Any() != true) + // Last resort: use the first filtered disk's preferred access path. + // GetPreferredAccessPath throws when the disk has no eligible volumes + // (e.g. a raw device consumed by LVM), so we catch and continue. + if (string.IsNullOrEmpty(accessPath)) + { + try { - throw new WorkloadException( - "Expected disks to test not found. Given the parameters defined for the profile action/step or those passed " + - "in on the command line, the requisite disks do not exist on the system or could not be identified based on the properties " + - "of the existing disks.", - ErrorReason.DependencyNotFound); + accessPath = filteredDisks.FirstOrDefault()?.GetPreferredAccessPath(this.Platform); } - - foreach (Disk disk in disksToTest) + catch (Exception) { - string mysqlPath = this.Combine(disk.GetPreferredAccessPath(this.Platform), "mysql"); - - // Create the directory if it doesn't exist - if (!this.SystemManager.FileSystem.Directory.Exists(mysqlPath)) - { - this.SystemManager.FileSystem.Directory.CreateDirectory(mysqlPath); - } - - diskPaths += $"{mysqlPath};"; + // The disk may not have any eligible volumes. } } - return diskPaths.TrimEnd(';'); + if (raidAccessPath != null) + { + this.Logger.LogTraceMessage($"{this.TypeName}: Found raid0 access path '{raidAccessPath}'."); + } + else + { + this.Logger.LogTraceMessage($"{this.TypeName}: No raid0 access path found. Using fallback access path '{accessPath}'."); + } + + if (string.IsNullOrEmpty(accessPath)) + { + throw new DependencyException( + "Expected disks not found. Given the parameters defined for the profile action/step or those passed " + + "in on the command line, the requisite disks do not exist on the system or could not be identified based on the properties " + + "of the existing disks.", + ErrorReason.DependencyNotFound); + } + + string mysqlPath = this.Combine(accessPath, "mysql"); + + if (!this.SystemManager.FileSystem.Directory.Exists(mysqlPath)) + { + this.Logger.LogTraceMessage($"{this.TypeName}: Creating MySQL InnoDB directory '{mysqlPath}'."); + this.SystemManager.FileSystem.Directory.CreateDirectory(mysqlPath); + } + + this.Logger.LogTraceMessage($"{this.TypeName}: MySQL InnoDB directory resolved to '{mysqlPath}'."); + + return mysqlPath; } private async Task GetMySQLInMemoryCapacityAsync(CancellationToken cancellationToken) diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index 9347e89900..9e3e9f63c6 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -22,6 +22,7 @@ namespace VirtualClient.Dependencies /// /// Provides functionality for configuring PostgreSQL Server. /// + [SupportedPlatforms("linux-x64,linux-arm64")] public class PostgreSQLServerConfiguration : ExecuteCommand { private const string PythonCommand = "python3"; @@ -29,7 +30,7 @@ public class PostgreSQLServerConfiguration : ExecuteCommand private string packageDirectory; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// An enumeration of dependencies that can be used for dependency injection. /// A series of key value pairs that dictate runtime execution. @@ -72,7 +73,6 @@ public string DiskFilter { get { - // and 256G return this.Parameters.GetValue(nameof(this.DiskFilter), "osdisk:false&sizegreaterthan:256g"); } } @@ -136,7 +136,6 @@ public string SharedMemoryBuffer /// A token that can be used to cancel the operation. protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) { - ProcessManager manager = this.SystemManager.ProcessManager; string stateId = $"{nameof(PostgreSQLServerConfiguration)}-{this.Action}-action-success"; ConfigurationState configurationState = await this.stateManager.GetStateAsync(stateId, cancellationToken) .ConfigureAwait(false); @@ -176,7 +175,9 @@ private async Task ConfigurePostgreSQLServerAsync(EventContext telemetryContext, .FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); - string arguments = $"{this.packageDirectory}/configure-server.py --dbName {this.DatabaseName} --serverIp {serverIp} --password {this.SuperUserPassword} --port {this.Port} --inMemory {this.SharedMemoryBuffer}"; + string directory = await this.GetPostgreSQLDataDirectoryAsync(cancellationToken); + + string arguments = $"{this.packageDirectory}/configure-server.py --dbName {this.DatabaseName} --serverIp {serverIp} --password {this.SuperUserPassword} --port {this.Port} --inMemory {this.SharedMemoryBuffer} --directory {directory}"; using (IProcessProxy process = await this.ExecuteCommandAsync( "python3", @@ -193,6 +194,82 @@ private async Task ConfigurePostgreSQLServerAsync(EventContext telemetryContext, } } + private async Task GetPostgreSQLDataDirectoryAsync(CancellationToken cancellationToken) + { + IEnumerable disks = await this.SystemManager.DiskManager.GetDisksAsync(cancellationToken) + .ConfigureAwait(false); + + IEnumerable filteredDisks = DiskFilters.FilterDisks(disks, this.DiskFilter, this.Platform); + + // Search ALL disks for a raid0 mount point. After StripeDisks creates an LVM + // striped volume from the filtered physical disks, the mount point appears on + // the logical volume device, which is a separate disk entry from the originals. + string raidAccessPath = disks + .SelectMany(d => d.Volumes) + .SelectMany(v => v.AccessPaths) + .FirstOrDefault(p => p.Contains("raid0", StringComparison.OrdinalIgnoreCase)); + + // lshw may not report LVM logical volumes at all. Fall back to reading + // /proc/mounts which always lists every active mount point. + if (string.IsNullOrEmpty(raidAccessPath) && this.Platform != PlatformID.Win32NT) + { + try + { + string procMounts = await this.SystemManager.FileSystem.File.ReadAllTextAsync("/proc/mounts", cancellationToken) + .ConfigureAwait(false); + + foreach (string line in procMounts.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + string[] parts = line.Split(' '); + if (parts.Length >= 2 && parts[1].Contains("raid0", StringComparison.OrdinalIgnoreCase)) + { + raidAccessPath = parts[1]; + break; + } + } + } + catch (Exception) + { + // /proc/mounts may not be available. + } + } + + string accessPath = raidAccessPath; + + // Last resort: use the first filtered disk's preferred access path. + // GetPreferredAccessPath throws when the disk has no eligible volumes + // (e.g. a raw device consumed by LVM), so we catch and continue. + if (string.IsNullOrEmpty(accessPath)) + { + try + { + accessPath = filteredDisks.FirstOrDefault()?.GetPreferredAccessPath(this.Platform); + } + catch (Exception) + { + // The disk may not have any eligible volumes. + } + } + + if (string.IsNullOrEmpty(accessPath)) + { + throw new DependencyException( + "Expected disks not found. Given the parameters defined for the profile action/step or those passed " + + "in on the command line, the requisite disks do not exist on the system or could not be identified based on the properties " + + "of the existing disks.", + ErrorReason.DependencyNotFound); + } + + string directory = this.Combine(accessPath, "postgresql"); + + if (!this.SystemManager.FileSystem.Directory.Exists(directory)) + { + this.SystemManager.FileSystem.Directory.CreateDirectory(directory); + } + + return directory; + } + private async Task SetupPostgreSQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken) { string arguments = $"{this.packageDirectory}/setup-database.py --dbName {this.DatabaseName} --password {this.SuperUserPassword} --port {this.Port}"; diff --git a/src/VirtualClient/VirtualClient.Dependencies/StripeDisks.cs b/src/VirtualClient/VirtualClient.Dependencies/StripeDisks.cs index c2d4108ec3..f5fb351dd6 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/StripeDisks.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/StripeDisks.cs @@ -137,6 +137,20 @@ protected override async Task InitializeAsync(EventContext telemetryContext, Can /// protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) { + // If the mount directory already exists and has content (e.g. from a previous run), + // skip the striping operation to avoid destroying existing data. + if (!string.IsNullOrEmpty(this.MountDirectory) + && this.fileSystem.Directory.Exists(this.MountDirectory) + && this.fileSystem.Directory.EnumerateFileSystemEntries(this.MountDirectory).Any()) + { + this.Logger.LogTraceMessage($"{this.TypeName}: Skipping. Mount directory '{this.MountDirectory}' already exists and contains data."); + telemetryContext.AddContext("skipped", true); + telemetryContext.AddContext("mountDirectory", this.MountDirectory); + return; + } + + // Discover all disks on the system and apply the user-specified filter + // (e.g. "osdisk:false&sizegreaterthan:256g") to identify eligible disks. IEnumerable allDisks = await this.systemManagement.DiskManager.GetDisksAsync(cancellationToken); IEnumerable filteredDisks = DiskFilters.FilterDisks(allDisks, this.DiskFilter, this.Platform); @@ -147,6 +161,7 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel ErrorReason.DependencyNotFound); } + // When DiskCount is specified, limit to that many disks (largest first). if (this.DiskCount > 0) { if (filteredDisks.Count() < this.DiskCount) @@ -162,11 +177,15 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel .Take(this.DiskCount); } + // Build the comma-separated list of device paths (e.g. "/dev/sdc,/dev/sdd") + // to pass to the platform-specific striping script. string diskPaths = string.Join(",", filteredDisks.Select(d => d.DevicePath)); string command; string commandArguments; + // On Windows the script is a .cmd batch file; on Linux it is a bash script + // that also receives the resolved mount directory for the striped volume. if (this.Platform == PlatformID.Win32NT) { command = "cmd"; @@ -182,6 +201,8 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel .AddContext("command", command) .AddContext("commandArguments", commandArguments); + // Execute the script with elevated privileges. The script handles partitioning, + // LVM striping (for multiple disks), filesystem creation, and mounting. using (IProcessProxy process = await this.ExecuteCommandAsync(command, commandArguments, this.ScriptDirectory, telemetryContext, cancellationToken, runElevated: true)) { if (!cancellationToken.IsCancellationRequested) @@ -193,7 +214,7 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel } /// - /// Resolves the mount directory path using the same logic as . + /// Resolves the mount directory path using the same convention as . /// Uses if provided, otherwise determines the path based on /// the current platform and logged-in user. /// diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json index 90fd0851e0..6e05b12f0a 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json @@ -154,13 +154,17 @@ "Type": "LinuxPackageInstallation", "Parameters": { "Scenario": "InstallLinuxPackages", - "Packages": "python3,python3-pyudev,python3-psutil" + "Packages": "python3" } }, { - "Type": "FormatDisks", + "Type": "DependencyPackageInstallation", "Parameters": { - "Scenario": "FormatDisks", + "Scenario": "DownloadSystemConfigPackage", + "BlobContainer": "packages", + "BlobName": "system_config.1.1.0.zip", + "PackageName": "system_config", + "Extract": true, "Role": "Server" } }, @@ -168,6 +172,8 @@ "Type": "StripeDisks", "Parameters": { "Scenario": "StripeAndMountDisks", + "PackageName": "system_config", + "DiskFilter": "$.Parameters.DiskFilter", "Role": "Server" } }, @@ -176,7 +182,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql-server-8.0.36.rev0.zip", + "BlobName": "mysql-server-8.0.36-v5.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" @@ -187,7 +193,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev3.zip", + "BlobName": "sysbench-1.0.20.rev5.zip", "PackageName": "sysbench", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json index d616771b37..cdf67d57aa 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json @@ -42,13 +42,17 @@ "Type": "LinuxPackageInstallation", "Parameters": { "Scenario": "InstallLinuxPackages", - "Packages": "python3,python3-pyudev,python3-psutil" + "Packages": "python3" } }, { - "Type": "FormatDisks", + "Type": "DependencyPackageInstallation", "Parameters": { - "Scenario": "FormatDisks", + "Scenario": "DownloadSystemConfigPackage", + "BlobContainer": "packages", + "BlobName": "system_config.1.1.0.zip", + "PackageName": "system_config", + "Extract": true, "Role": "Server" } }, @@ -56,6 +60,8 @@ "Type": "StripeDisks", "Parameters": { "Scenario": "StripeAndMountDisks", + "PackageName": "system_config", + "DiskFilter": "$.Parameters.DiskFilter", "Role": "Server" } }, @@ -64,7 +70,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql-server-8.0.36.rev0.zip", + "BlobName": "mysql-server-8.0.36-v5.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" @@ -75,7 +81,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev3.zip", + "BlobName": "sysbench-1.0.20.rev5.zip", "PackageName": "sysbench", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json index da7ec886d3..832be21a0a 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json @@ -1,5 +1,5 @@ { - "Description": "HammerDB PostgrSQL TPCC Database Server Performance Workload", + "Description": "HammerDB PostgreSQL TPCC Database Server Performance Workload", "MinimumExecutionInterval": "00:01:00", "Metadata": { "RecommendedMinimumExecutionTime": "04:00:00", @@ -12,7 +12,7 @@ "Port": "5432", "Duration": "00:20:00", "VirtualUsers": "{calculate({LogicalCoreCount})}", - "WarehouseCount": "{calculate({SystemMemoryMegabytes} * 15 / 800)}", + "WarehouseCount": "{calculate({LogicalCoreCount} * 10)}", "SharedMemoryBuffer": "{calculate({SystemMemoryMegabytes} * 85 / 100)}" }, "Actions": [ @@ -49,24 +49,30 @@ ], "Dependencies": [ { - "Type": "FormatDisks", + "Type": "LinuxPackageInstallation", "Parameters": { - "Scenario": "FormatDisks", - "Role": "Server" + "Scenario": "InstallLinuxPackages", + "Packages": "python3" } }, { - "Type": "MountDisks", + "Type": "DependencyPackageInstallation", "Parameters": { - "Scenario": "CreateMountPoints", + "Scenario": "DownloadSystemConfigPackage", + "BlobContainer": "packages", + "BlobName": "system_config.1.1.0.zip", + "PackageName": "system_config", + "Extract": true, "Role": "Server" } }, { - "Type": "LinuxPackageInstallation", + "Type": "StripeDisks", "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" + "Scenario": "StripeAndMountDisks", + "PackageName": "system_config", + "DiskFilter": "$.Parameters.DiskFilter", + "Role": "Server" } }, { @@ -74,7 +80,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev3.zip", + "BlobName": "postgresql.14.0.0.rev4.zip", "PackageName": "postgresql", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json index e077610d55..82bbd23175 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCH.json @@ -12,7 +12,7 @@ "Port": "5432", "Duration": "00:20:00", "VirtualUsers": "{calculate({LogicalCoreCount})}", - "ScaleFactor": "10", + "ScaleFactor": "1", "SharedMemoryBuffer": "{calculate({SystemMemoryMegabytes} * 85 / 100)}" }, "Actions": [ @@ -49,24 +49,30 @@ ], "Dependencies": [ { - "Type": "FormatDisks", + "Type": "LinuxPackageInstallation", "Parameters": { - "Scenario": "FormatDisks", - "Role": "Server" + "Scenario": "InstallLinuxPackages", + "Packages": "python3" } }, { - "Type": "MountDisks", + "Type": "DependencyPackageInstallation", "Parameters": { - "Scenario": "CreateMountPoints", + "Scenario": "DownloadSystemConfigPackage", + "BlobContainer": "packages", + "BlobName": "system_config.1.1.0.zip", + "PackageName": "system_config", + "Extract": true, "Role": "Server" } }, { - "Type": "LinuxPackageInstallation", + "Type": "StripeDisks", "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" + "Scenario": "StripeAndMountDisks", + "PackageName": "system_config", + "DiskFilter": "$.Parameters.DiskFilter", + "Role": "Server" } }, { @@ -74,7 +80,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev3.zip", + "BlobName": "postgresql.14.0.0.rev4.zip", "PackageName": "postgresql", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json index a4c012ffef..8fd94e04e5 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json @@ -152,16 +152,29 @@ ], "Dependencies": [ { - "Type": "FormatDisks", + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages": "python3" + } + }, + { + "Type": "DependencyPackageInstallation", "Parameters": { - "Scenario": "FormatDisks", + "Scenario": "DownloadSystemConfigPackage", + "BlobContainer": "packages", + "BlobName": "system_config.1.1.0.zip", + "PackageName": "system_config", + "Extract": true, "Role": "Server" } }, { - "Type": "MountDisks", + "Type": "StripeDisks", "Parameters": { - "Scenario": "CreateMountPoints", + "Scenario": "StripeAndMountDisks", + "PackageName": "system_config", + "DiskFilter": "$.Parameters.DiskFilter", "Role": "Server" } }, @@ -181,18 +194,11 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev3.zip", + "BlobName": "sysbench-1.0.20.rev5.zip", "PackageName": "sysbench", "Extract": true } }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" - } - }, { "Type": "PostgreSQLServerInstallation", "Parameters": { @@ -226,31 +232,6 @@ "SharedMemoryBuffer": "$.Parameters.SharedMemoryBuffer" } }, - { - "Type": "SysbenchConfiguration", - "Parameters": { - "Scenario": "PreparePostgreSQLDatabase", - "Action": "CreateTables", - "DatabaseSystem": "PostgreSQL", - "Benchmark": "OLTP", - "DatabaseName": "$.Parameters.DatabaseName", - "DatabaseScenario": "$.Parameters.DatabaseScenario", - "PackageName": "sysbench", - "Role": "Server" - } - }, - { - "Type": "PostgreSQLServerConfiguration", - "Parameters": { - "Scenario": "DistributePostgreSQLDatabase", - "Action": "DistributeDatabase", - "DatabaseName": "$.Parameters.DatabaseName", - "DiskFilter": "$.Parameters.DiskFilter", - "PackageName": "postgresql", - "Port": "$.Parameters.Port", - "Role": "Server" - } - }, { "Type": "SysbenchConfiguration", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json index 5de85fd32c..8b7ac7a628 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json @@ -1,5 +1,5 @@ { - "Description": "Sysbench TPCC MySQL Database Server Performance Workload", + "Description": "Sysbench TPCC PostgreSQL Database Server Performance Workload", "MinimumExecutionInterval": "00:01:00", "Metadata": { "RecommendedMinimumExecutionTime": "04:00:00", @@ -40,16 +40,29 @@ ], "Dependencies": [ { - "Type": "FormatDisks", + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages": "python3" + } + }, + { + "Type": "DependencyPackageInstallation", "Parameters": { - "Scenario": "FormatDisks", + "Scenario": "DownloadSystemConfigPackage", + "BlobContainer": "packages", + "BlobName": "system_config.1.1.0.zip", + "PackageName": "system_config", + "Extract": true, "Role": "Server" } }, { - "Type": "MountDisks", + "Type": "StripeDisks", "Parameters": { - "Scenario": "CreateMountPoints", + "Scenario": "StripeAndMountDisks", + "PackageName": "system_config", + "DiskFilter": "$.Parameters.DiskFilter", "Role": "Server" } }, @@ -69,18 +82,11 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev3.zip", + "BlobName": "sysbench-1.0.20.rev5.zip", "PackageName": "sysbench", "Extract": true } }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" - } - }, { "Type": "PostgreSQLServerInstallation", "Parameters": { @@ -114,31 +120,6 @@ "SharedMemoryBuffer": "$.Parameters.SharedMemoryBuffer" } }, - { - "Type": "SysbenchConfiguration", - "Parameters": { - "Scenario": "PreparePostgreSQLDatabase", - "Action": "CreateTables", - "DatabaseSystem": "PostgreSQL", - "Benchmark": "TPCC", - "DatabaseName": "$.Parameters.DatabaseName", - "DatabaseScenario": "$.Parameters.DatabaseScenario", - "PackageName": "sysbench", - "Role": "Server" - } - }, - { - "Type": "PostgreSQLServerConfiguration", - "Parameters": { - "Scenario": "DistributePostgreSQLDatabase", - "Action": "DistributeDatabase", - "DatabaseName": "$.Parameters.DatabaseName", - "DiskFilter": "$.Parameters.DiskFilter", - "PackageName": "postgresql", - "Port": "$.Parameters.Port", - "Role": "Server" - } - }, { "Type": "SysbenchConfiguration", "Parameters": {