Skip to content

[tests] Skip PerformanceTest on slow CI machines using evaluation time #11249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
simonrozsival merged 2 commits into from
May 1, 2026
+32 −11
Merged

[tests] Skip PerformanceTest on slow CI machines using evaluation time #11249

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
95111b5
[tests] Skip PerformanceTest on slow CI machines using evaluation time
jonathanpeppers Apr 29, 2026
98726c2
Fix CS0118: use alias to disambiguate Build type from namespace
jonathanpeppers Apr 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 32 additions & 11 deletions tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Runtime.CompilerServices;
using Microsoft.Build.Framework;
using Microsoft.Build.Logging.StructuredLogger;
using StructuredBuild = Microsoft.Build.Logging.StructuredLogger.Build;
using NUnit.Framework;
using Xamarin.Android.Tasks;
using Xamarin.Android.Tools;
Expand All @@ -18,6 +19,7 @@ namespace Xamarin.Android.Build.Tests
public class PerformanceTest : DeviceTest
{
const int Retry = 2;
const int MaxEvaluationTimeInMs = 500;
static readonly Dictionary<string, int> csv_values = new Dictionary<string, int> ();

[OneTimeSetUp]
Expand Down Expand Up @@ -60,9 +62,11 @@ void Profile (ProjectBuilder builder, int iterations, Action<ProjectBuilder> act
}

double total = 0;
StructuredBuild build = null;
for (int i=0; i < iterations; i++) {
action (builder);
var actual = GetDurationFromBinLog (builder);
build = ReadBinLog (builder);
var actual = GetDuration (build, builder);
TestContext.Out.WriteLine ($"run {i} took: {actual}ms");
total += actual;
if (afterRun is not null)
Expand All @@ -71,6 +75,7 @@ void Profile (ProjectBuilder builder, int iterations, Action<ProjectBuilder> act
total /= iterations;
TestContext.Out.WriteLine ($"expected: {expected}ms, actual: {total}ms");
if (total > expected) {
AssertSlowMachine (build, expected, total);
Assert.Fail ($"Exceeded expected time of {expected}ms, actual {total}ms");
Comment thread
jonathanpeppers marked this conversation as resolved.
}
}
Expand All @@ -81,47 +86,63 @@ void ProfileTask (ProjectBuilder builder, string task, int iterations, Action<Pr
Assert.Fail ($"No timeout value found for a key of {caller}");
}
double total = 0;
StructuredBuild build = null;
for (int i=0; i < iterations; i++) {
action (builder);
var duration = GetTaskDurationFromBinLog (builder, task);
build = ReadBinLog (builder);
var duration = GetTaskDuration (build, builder, task);
TestContext.Out.WriteLine($"run {i} took: {duration}ms");
total += duration;
}
total /= iterations;
TestContext.Out.WriteLine($"expected: {expected}ms, actual: {total}ms");
if (total > expected) {
AssertSlowMachine (build, expected, total);
Assert.Fail ($"Exceeded expected time of {expected}ms, actual {total}ms");
Comment thread
jonathanpeppers marked this conversation as resolved.
}
}

double GetTaskDurationFromBinLog (ProjectBuilder builder, string task)
void AssertSlowMachine (StructuredBuild build, int expected, double actual)
{
var evaluations = build.FindChildrenRecursive<ProjectEvaluation> ();
var maxEval = evaluations.Any () ? evaluations.Max (e => e.Duration.TotalMilliseconds) : 0;
TestContext.Out.WriteLine ($"max evaluation time: {maxEval}ms (threshold: {MaxEvaluationTimeInMs}ms)");
if (maxEval > MaxEvaluationTimeInMs) {
Assert.Inconclusive ($"Exceeded expected time of {expected}ms, actual {actual}ms, but evaluation time was {maxEval:F0}ms (threshold: {MaxEvaluationTimeInMs}ms), indicating a slow CI machine.");
}
}

StructuredBuild ReadBinLog (ProjectBuilder builder)
{
var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog");
FileAssert.Exists (binlog);
return BinaryLog.ReadBuild (binlog);
}

var build = BinaryLog.ReadBuild (binlog);
double GetTaskDuration (StructuredBuild build, ProjectBuilder builder, string task)
{
var duration = build
.FindChildrenRecursive<Task> (t => t.Name == task)
.Aggregate (TimeSpan.Zero, (duration, target) => duration + target.Duration);

if (duration == TimeSpan.Zero)
if (duration == TimeSpan.Zero) {
var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog");
throw new InvalidDataException ($"No task build duration found in {binlog}");
}

return duration.TotalMilliseconds;
}

double GetDurationFromBinLog (ProjectBuilder builder)
double GetDuration (StructuredBuild build, ProjectBuilder builder)
{
var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog");
FileAssert.Exists (binlog);

var build = BinaryLog.ReadBuild (binlog);
var duration = build
.FindChildrenRecursive<Project> ()
.Aggregate (TimeSpan.Zero, (duration, project) => duration + project.Duration);

if (duration == TimeSpan.Zero)
if (duration == TimeSpan.Zero) {
var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog");
throw new InvalidDataException ($"No project build duration found in {binlog}");
}

return duration.TotalMilliseconds;
}
Expand Down