From 95111b54a3258911bc3cd861d4e945e2409975c8 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Wed, 29 Apr 2026 15:03:09 -0500 Subject: [PATCH 1/2] [tests] Skip PerformanceTest on slow CI machines using evaluation time When a performance test exceeds its expected time, check the MSBuild project evaluation duration from the binlog. If evaluation alone exceeds 500ms (normal is ~200-350ms), the CI machine is too slow for reliable performance measurements. Use Assert.Inconclusive() instead of Assert.Fail() so the test is marked as skipped rather than failed. Analysis of 10 recent CI failures showed evaluation times of 558-854ms on slow machines, while normal machines should be well under 500ms. Also refactored binlog reading so BinaryLog.ReadBuild() is called once per iteration, with the Build object passed to helper methods. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/PerformanceTest.cs | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs index 58597ec4c70..779157de310 100644 --- a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs @@ -18,6 +18,7 @@ namespace Xamarin.Android.Build.Tests public class PerformanceTest : DeviceTest { const int Retry = 2; + const int MaxEvaluationTimeInMs = 500; static readonly Dictionary csv_values = new Dictionary (); [OneTimeSetUp] @@ -60,9 +61,11 @@ void Profile (ProjectBuilder builder, int iterations, Action act } double total = 0; + Build build = null; for (int i=0; i < iterations; i++) { action (builder); - var actual = GetDurationFromBinLog (builder); + build = ReadBinLog (builder); + var actual = GetDuration (build, builder); TestContext.Out.WriteLine ($"run {i} took: {actual}ms"); total += actual; if (afterRun is not null) @@ -71,6 +74,7 @@ void Profile (ProjectBuilder builder, int iterations, Action act total /= iterations; TestContext.Out.WriteLine ($"expected: {expected}ms, actual: {total}ms"); if (total > expected) { + AssertSlowMachine (build, expected, total); Assert.Fail ($"Exceeded expected time of {expected}ms, actual {total}ms"); } } @@ -81,47 +85,63 @@ void ProfileTask (ProjectBuilder builder, string task, int iterations, Action expected) { + AssertSlowMachine (build, expected, total); Assert.Fail ($"Exceeded expected time of {expected}ms, actual {total}ms"); } } - double GetTaskDurationFromBinLog (ProjectBuilder builder, string task) + void AssertSlowMachine (Build build, int expected, double actual) + { + var evaluations = build.FindChildrenRecursive (); + var maxEval = evaluations.Any () ? evaluations.Max (e => e.Duration.TotalMilliseconds) : 0; + TestContext.Out.WriteLine ($"max evaluation time: {maxEval}ms (threshold: {MaxEvaluationTimeInMs}ms)"); + if (maxEval > MaxEvaluationTimeInMs) { + Assert.Inconclusive ($"Exceeded expected time of {expected}ms, actual {actual}ms, but evaluation time was {maxEval:F0}ms (threshold: {MaxEvaluationTimeInMs}ms), indicating a slow CI machine."); + } + } + + Build ReadBinLog (ProjectBuilder builder) { var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); FileAssert.Exists (binlog); + return BinaryLog.ReadBuild (binlog); + } - var build = BinaryLog.ReadBuild (binlog); + double GetTaskDuration (Build build, ProjectBuilder builder, string task) + { var duration = build .FindChildrenRecursive (t => t.Name == task) .Aggregate (TimeSpan.Zero, (duration, target) => duration + target.Duration); - if (duration == TimeSpan.Zero) + if (duration == TimeSpan.Zero) { + var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); throw new InvalidDataException ($"No task build duration found in {binlog}"); + } return duration.TotalMilliseconds; } - double GetDurationFromBinLog (ProjectBuilder builder) + double GetDuration (Build build, ProjectBuilder builder) { - var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); - FileAssert.Exists (binlog); - - var build = BinaryLog.ReadBuild (binlog); var duration = build .FindChildrenRecursive () .Aggregate (TimeSpan.Zero, (duration, project) => duration + project.Duration); - if (duration == TimeSpan.Zero) + if (duration == TimeSpan.Zero) { + var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); throw new InvalidDataException ($"No project build duration found in {binlog}"); + } return duration.TotalMilliseconds; } From 98726c2fc26a24592b318b4c5a58569f424d844c Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 30 Apr 2026 08:59:09 -0500 Subject: [PATCH 2/2] Fix CS0118: use alias to disambiguate Build type from namespace Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/PerformanceTest.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs index 779157de310..e867dfb13ac 100644 --- a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs @@ -5,6 +5,7 @@ using System.Runtime.CompilerServices; using Microsoft.Build.Framework; using Microsoft.Build.Logging.StructuredLogger; +using StructuredBuild = Microsoft.Build.Logging.StructuredLogger.Build; using NUnit.Framework; using Xamarin.Android.Tasks; using Xamarin.Android.Tools; @@ -61,7 +62,7 @@ void Profile (ProjectBuilder builder, int iterations, Action act } double total = 0; - Build build = null; + StructuredBuild build = null; for (int i=0; i < iterations; i++) { action (builder); build = ReadBinLog (builder); @@ -85,7 +86,7 @@ void ProfileTask (ProjectBuilder builder, string task, int iterations, Action (); var maxEval = evaluations.Any () ? evaluations.Max (e => e.Duration.TotalMilliseconds) : 0; @@ -111,14 +112,14 @@ void AssertSlowMachine (Build build, int expected, double actual) } } - Build ReadBinLog (ProjectBuilder builder) + StructuredBuild ReadBinLog (ProjectBuilder builder) { var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); FileAssert.Exists (binlog); return BinaryLog.ReadBuild (binlog); } - double GetTaskDuration (Build build, ProjectBuilder builder, string task) + double GetTaskDuration (StructuredBuild build, ProjectBuilder builder, string task) { var duration = build .FindChildrenRecursive (t => t.Name == task) @@ -132,7 +133,7 @@ double GetTaskDuration (Build build, ProjectBuilder builder, string task) return duration.TotalMilliseconds; } - double GetDuration (Build build, ProjectBuilder builder) + double GetDuration (StructuredBuild build, ProjectBuilder builder) { var duration = build .FindChildrenRecursive ()