From acf6e29484646cbec9fdf78e2b94a9f41cace2e2 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Fri, 26 Jun 2026 14:24:30 +0200 Subject: [PATCH 1/5] feat: avoid final field modifications in karate instrumentation --- .../karate/ExecutionContext.java | 22 ++++++++++ .../KarateExecutionInstrumentation.java | 44 +++++++++++++++++-- .../instrumentation/karate/KarateUtils.java | 7 --- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java index fbcc6c37d90..73b5b2159bd 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java @@ -1,6 +1,7 @@ package datadog.trace.instrumentation.karate; import com.intuit.karate.core.Scenario; +import com.intuit.karate.core.ScenarioResult; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.execution.TestExecutionPolicy; @@ -10,6 +11,8 @@ public class ExecutionContext { private final TestExecutionPolicy executionPolicy; private boolean suppressFailures; + private ScenarioResult originalResult; + private ScenarioResult finalResult; public ExecutionContext(TestExecutionPolicy executionPolicy) { this.executionPolicy = executionPolicy; @@ -29,6 +32,25 @@ public TestExecutionPolicy getExecutionPolicy() { return executionPolicy; } + /** + * Records the canonical result for a retried scenario. Used to check whether the result must be + * substituted with the result of the final attempt. + */ + public void recordResultSubstitution(ScenarioResult originalResult, ScenarioResult finalResult) { + this.originalResult = originalResult; + this.finalResult = finalResult; + } + + /** + * Returns the result that should replace the supplied one when it is added to the {@code + * FeatureResult}, or {@code null} if no substitution applies. This repoints the canonical + * scenario result to the final retry attempt without mutating {@code ScenarioRuntime#result}, + * which is not JEP 500 compliant. + */ + public ScenarioResult substituteResult(ScenarioResult result) { + return result == originalResult ? finalResult : null; + } + public static ExecutionContext create(Scenario scenario) { TestIdentifier testIdentifier = KarateUtils.toTestIdentifier(scenario); Collection testTags = scenario.getTagsEffective().getTagKeys(); diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java index f11b647d93e..635d7975b77 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java @@ -38,7 +38,9 @@ public boolean isEnabled() { @Override public String[] knownMatchingTypes() { return new String[] { - "com.intuit.karate.core.ScenarioRuntime", "com.intuit.karate.core.ScenarioResult" + "com.intuit.karate.core.ScenarioRuntime", + "com.intuit.karate.core.ScenarioResult", + "com.intuit.karate.core.FeatureResult" }; } @@ -71,6 +73,13 @@ public void methodAdvice(MethodTransformer transformer) { .and(takesArguments(1)) .and(takesArgument(0, named("com.intuit.karate.core.StepResult"))), KarateExecutionInstrumentation.class.getName() + "$SuppressErrorAdvice"); + + // FeatureResult + transformer.applyAdvice( + named("addResult") + .and(takesArguments(1)) + .and(takesArgument(0, named("com.intuit.karate.core.ScenarioResult"))), + KarateExecutionInstrumentation.class.getName() + "$ResultSubstitutionAdvice"); } public static class RetryAdvice { @@ -103,7 +112,8 @@ public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { return; } - ScenarioResult finalResult = scenarioRuntime.result; + ScenarioResult originalResult = scenarioRuntime.result; + ScenarioResult finalResult = originalResult; TestExecutionPolicy executionPolicy = context.getExecutionPolicy(); while (executionPolicy.applicable()) { @@ -115,7 +125,12 @@ public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { finalResult = retry.result; } - KarateUtils.setResult(scenarioRuntime, finalResult); + // The original runtime's result is registered with the FeatureResult by Karate once this + // method returns. Record the substitution so ResultSubstitutionAdvice repoints that canonical + // entry to the final attempt, instead of mutating the final ScenarioRuntime#result field. + if (finalResult != originalResult) { + context.recordResultSubstitution(originalResult, finalResult); + } CallDepthThreadLocalMap.reset(ScenarioRuntime.class); } @@ -153,4 +168,27 @@ public static void muzzleCheck(RuntimeHook runtimeHook) { runtimeHook.beforeSuite(null); } } + + public static class ResultSubstitutionAdvice { + @Advice.OnMethodEnter + public static void onAddResult( + @Advice.Argument(value = 0, readOnly = false) ScenarioResult result) { + ExecutionContext context = + InstrumentationContext.get(Scenario.class, ExecutionContext.class) + .get(result.getScenario()); + if (context == null) { + return; + } + + ScenarioResult substitute = context.substituteResult(result); + if (substitute != null) { + result = substitute; + } + } + + // Karate 1.0.0 and above + public static void muzzleCheck(RuntimeHook runtimeHook) { + runtimeHook.beforeSuite(null); + } + } } diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java index 80b422d9f59..1525d5dd741 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateUtils.java @@ -7,7 +7,6 @@ import com.intuit.karate.core.FeatureRuntime; import com.intuit.karate.core.Result; import com.intuit.karate.core.Scenario; -import com.intuit.karate.core.ScenarioResult; import com.intuit.karate.core.ScenarioRuntime; import com.intuit.karate.core.Tag; import datadog.trace.api.civisibility.config.LibraryCapability; @@ -46,8 +45,6 @@ private KarateUtils() {} // static method to create aborted result has a different signature starting with Karate 1.4.1 private static final MethodHandle ABORTED_RESULT_STARTTIME_DURATION_NANOS = METHOD_HANDLES.method(Result.class, "aborted", long.class, long.class); - private static final MethodHandle SCENARIO_RUNTIME_RESULT_SETTER = - METHOD_HANDLES.privateFieldSetter(ScenarioRuntime.class, "result"); private static final ComparableVersion karateV12 = new ComparableVersion("1.2.0"); private static final ComparableVersion karateV13 = new ComparableVersion("1.3.0"); @@ -153,10 +150,6 @@ public static void resetBeforeHook(FeatureRuntime featureRuntime) { METHOD_HANDLES.invoke(FEATURE_RUNTIME_BEFORE_HOOK_DONE_SETTER, featureRuntime, false); } - public static void setResult(ScenarioRuntime runtime, ScenarioResult result) { - METHOD_HANDLES.invoke(SCENARIO_RUNTIME_RESULT_SETTER, runtime, result); - } - public static String getKarateVersion() { return FileUtils.KARATE_VERSION; } From 277ad1a30e7461016905735a0580d4713349d5df Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Mon, 29 Jun 2026 14:06:26 +0200 Subject: [PATCH 2/5] fix: revert changes --- .../karate/ExecutionContext.java | 26 +------------- .../KarateExecutionInstrumentation.java | 34 +------------------ 2 files changed, 2 insertions(+), 58 deletions(-) diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java index 73b5b2159bd..cf909a85e89 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/ExecutionContext.java @@ -1,7 +1,6 @@ package datadog.trace.instrumentation.karate; import com.intuit.karate.core.Scenario; -import com.intuit.karate.core.ScenarioResult; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.execution.TestExecutionPolicy; @@ -11,8 +10,6 @@ public class ExecutionContext { private final TestExecutionPolicy executionPolicy; private boolean suppressFailures; - private ScenarioResult originalResult; - private ScenarioResult finalResult; public ExecutionContext(TestExecutionPolicy executionPolicy) { this.executionPolicy = executionPolicy; @@ -22,9 +19,7 @@ public void setSuppressFailures(boolean suppressFailures) { this.suppressFailures = suppressFailures; } - public boolean getAndResetSuppressFailures() { - boolean suppressFailures = this.suppressFailures; - this.suppressFailures = false; + public boolean shouldSuppressFailures() { return suppressFailures; } @@ -32,25 +27,6 @@ public TestExecutionPolicy getExecutionPolicy() { return executionPolicy; } - /** - * Records the canonical result for a retried scenario. Used to check whether the result must be - * substituted with the result of the final attempt. - */ - public void recordResultSubstitution(ScenarioResult originalResult, ScenarioResult finalResult) { - this.originalResult = originalResult; - this.finalResult = finalResult; - } - - /** - * Returns the result that should replace the supplied one when it is added to the {@code - * FeatureResult}, or {@code null} if no substitution applies. This repoints the canonical - * scenario result to the final retry attempt without mutating {@code ScenarioRuntime#result}, - * which is not JEP 500 compliant. - */ - public ScenarioResult substituteResult(ScenarioResult result) { - return result == originalResult ? finalResult : null; - } - public static ExecutionContext create(Scenario scenario) { TestIdentifier testIdentifier = KarateUtils.toTestIdentifier(scenario); Collection testTags = scenario.getTagsEffective().getTagKeys(); diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java index 635d7975b77..dd40e00dde5 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java @@ -38,9 +38,7 @@ public boolean isEnabled() { @Override public String[] knownMatchingTypes() { return new String[] { - "com.intuit.karate.core.ScenarioRuntime", - "com.intuit.karate.core.ScenarioResult", - "com.intuit.karate.core.FeatureResult" + "com.intuit.karate.core.ScenarioRuntime", "com.intuit.karate.core.ScenarioResult" }; } @@ -73,13 +71,6 @@ public void methodAdvice(MethodTransformer transformer) { .and(takesArguments(1)) .and(takesArgument(0, named("com.intuit.karate.core.StepResult"))), KarateExecutionInstrumentation.class.getName() + "$SuppressErrorAdvice"); - - // FeatureResult - transformer.applyAdvice( - named("addResult") - .and(takesArguments(1)) - .and(takesArgument(0, named("com.intuit.karate.core.ScenarioResult"))), - KarateExecutionInstrumentation.class.getName() + "$ResultSubstitutionAdvice"); } public static class RetryAdvice { @@ -168,27 +159,4 @@ public static void muzzleCheck(RuntimeHook runtimeHook) { runtimeHook.beforeSuite(null); } } - - public static class ResultSubstitutionAdvice { - @Advice.OnMethodEnter - public static void onAddResult( - @Advice.Argument(value = 0, readOnly = false) ScenarioResult result) { - ExecutionContext context = - InstrumentationContext.get(Scenario.class, ExecutionContext.class) - .get(result.getScenario()); - if (context == null) { - return; - } - - ScenarioResult substitute = context.substituteResult(result); - if (substitute != null) { - result = substitute; - } - } - - // Karate 1.0.0 and above - public static void muzzleCheck(RuntimeHook runtimeHook) { - runtimeHook.beforeSuite(null); - } - } } From eb735e2e3fcab8be031305eea9f95e8c5e7f65b5 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Mon, 29 Jun 2026 14:09:11 +0200 Subject: [PATCH 3/5] feat: propagate failed results to original --- ...ntation.testing-framework-tests.gradle.kts | 1 + .../KarateExecutionInstrumentation.java | 15 +- .../src/test/groovy/KarateTest.groovy | 13 +- .../src/test/java/org/example/Flaky.java | 8 + .../TestContinueOnStepFailureKarate.java | 11 + .../test_continue_on_step_failure.feature | 7 + .../coverages.ftl | 1 + .../events.ftl | 526 ++++++++++++++++++ 8 files changed, 571 insertions(+), 11 deletions(-) create mode 100644 dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/TestContinueOnStepFailureKarate.java create mode 100644 dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/test_continue_on_step_failure.feature create mode 100644 dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/coverages.ftl create mode 100644 dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/events.ftl diff --git a/buildSrc/src/main/kotlin/dd-trace-java.instrumentation.testing-framework-tests.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.instrumentation.testing-framework-tests.gradle.kts index 0f2af6fb44a..ab92a96baac 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.instrumentation.testing-framework-tests.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.instrumentation.testing-framework-tests.gradle.kts @@ -6,6 +6,7 @@ logger.info("Avoid executing classes used to test testing frameworks instrumenta tasks.withType().configureEach { exclude("**/TestAssumption*", "**/TestSuiteSetUpAssumption*") + exclude("**/TestContinueOnStepFailure*") exclude("**/TestDisableTestTrace*") exclude("**/TestError*") exclude("**/TestFactory*") diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java index dd40e00dde5..6666445f5ca 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java @@ -116,11 +116,11 @@ public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { finalResult = retry.result; } - // The original runtime's result is registered with the FeatureResult by Karate once this - // method returns. Record the substitution so ResultSubstitutionAdvice repoints that canonical - // entry to the final attempt, instead of mutating the final ScenarioRuntime#result field. - if (finalResult != originalResult) { - context.recordResultSubstitution(originalResult, finalResult); + // When the scenario was retried, the original runtime's result must reflect the final + // attempt's outcome. To avoid final field modifications, the final attempt's failure is reflected onto + // the original result via addStepResult + if (finalResult.isFailed() && !originalResult.isFailed()) { + originalResult.addStepResult(finalResult.getFailedStep()); } CallDepthThreadLocalMap.reset(ScenarioRuntime.class); @@ -146,7 +146,10 @@ public static void onAddingStepResult( return; } - if (executionContext.getAndResetSuppressFailures()) { + // Suppress every failing step of a to-be-retried attempt (not just the first): with + // continueOnStepFailure a single attempt can add multiple failing steps, and any leak would + // mark the original runtime's result failed + if (executionContext.shouldSuppressFailures()) { stepResult = new StepResult(stepResult.getStep(), KarateUtils.abortedResult()); stepResult.setFailedReason(result.getError()); stepResult.setErrorIgnored(true); diff --git a/dd-java-agent/instrumentation/karate-1.0/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate-1.0/src/test/groovy/KarateTest.groovy index 0ae24e826fc..442222a9963 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate-1.0/src/test/groovy/KarateTest.groovy @@ -72,11 +72,14 @@ class KarateTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | success | tests | retriedTests - "test-failed" | false | [TestFailedKarate] | [] - "test-retry-failed" | false | [TestFailedKarate] | [new TestFQN("[org/example/test_failed] test failed", "second scenario")] - "test-failed-then-succeed" | true | [TestFailedThenSucceedKarate] | [new TestFQN("[org/example/test_failed_then_succeed] test failed", "flaky scenario")] - "test-retry-parameterized" | false | [TestFailedParameterizedKarate] | [ + testcaseName | success | tests | retriedTests + "test-failed" | false | [TestFailedKarate] | [] + "test-retry-failed" | false | [TestFailedKarate] | [new TestFQN("[org/example/test_failed] test failed", "second scenario")] + "test-failed-then-succeed" | true | [TestFailedThenSucceedKarate] | [new TestFQN("[org/example/test_failed_then_succeed] test failed", "flaky scenario")] + "test-retry-continue-on-step-failure" | true | [TestContinueOnStepFailureKarate] | [ + new TestFQN("[org/example/test_continue_on_step_failure] test continue on step failure", "flaky scenario") + ] + "test-retry-parameterized" | false | [TestFailedParameterizedKarate] | [ new TestFQN("[org/example/test_failed_parameterized] test parameterized", "first scenario as an outline") ] } diff --git a/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/Flaky.java b/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/Flaky.java index 59b938aaf57..2a474c24299 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/Flaky.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/Flaky.java @@ -5,8 +5,16 @@ public class Flaky { private static int counter = 0; + private static int stepCounter = 0; + // Fails the first two attempts, passes from the third onwards. public static void flake() { assertTrue(++counter >= 3); } + + // Same flaky behavior exposed as a value, for continueOnStepFailure scenarios that assert it in + // several steps. Uses a separate counter because both helpers run in the same test JVM. + public static boolean shouldPass() { + return ++stepCounter >= 3; + } } diff --git a/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/TestContinueOnStepFailureKarate.java b/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/TestContinueOnStepFailureKarate.java new file mode 100644 index 00000000000..7aa9d4c435a --- /dev/null +++ b/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/TestContinueOnStepFailureKarate.java @@ -0,0 +1,11 @@ +package org.example; + +import com.intuit.karate.junit5.Karate; + +public class TestContinueOnStepFailureKarate { + + @Karate.Test + public Karate test() { + return Karate.run("classpath:org/example/test_continue_on_step_failure.feature"); + } +} diff --git a/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/test_continue_on_step_failure.feature b/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/test_continue_on_step_failure.feature new file mode 100644 index 00000000000..83fe25d5f84 --- /dev/null +++ b/dd-java-agent/instrumentation/karate-1.0/src/test/java/org/example/test_continue_on_step_failure.feature @@ -0,0 +1,7 @@ +Feature: test continue on step failure + + Scenario: flaky scenario + * configure continueOnStepFailure = { enabled: true, continueAfter: true } + * def pass = Java.type('org.example.Flaky').shouldPass() + * match pass == true + * match pass == true diff --git a/dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/coverages.ftl b/dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/events.ftl b/dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/events.ftl new file mode 100644 index 00000000000..15752c0fcb1 --- /dev/null +++ b/dd-java-agent/instrumentation/karate-1.0/src/test/resources/test-retry-continue-on-step-failure/events.ftl @@ -0,0 +1,526 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* configure continueOnStepFailure = { enabled: true, continueAfter: true }" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* configure continueOnStepFailure = { enabled: true, continueAfter: true }", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* configure continueOnStepFailure = { enabled: true, continueAfter: true }" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_2}, + "resource" : "* configure continueOnStepFailure = { enabled: true, continueAfter: true }", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_2}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* configure continueOnStepFailure = { enabled: true, continueAfter: true }" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_3}, + "resource" : "* configure continueOnStepFailure = { enabled: true, continueAfter: true }", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_3}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* def pass = Java.type('org.example.Flaky').shouldPass()" + }, + "metrics" : { + "step.endLine" : 5, + "step.startLine" : 5 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* def pass = Java.type('org.example.Flaky').shouldPass()", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_4}, + "start" : ${content_start_4}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* def pass = Java.type('org.example.Flaky').shouldPass()" + }, + "metrics" : { + "step.endLine" : 5, + "step.startLine" : 5 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_2}, + "resource" : "* def pass = Java.type('org.example.Flaky').shouldPass()", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_5}, + "start" : ${content_start_5}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* def pass = Java.type('org.example.Flaky').shouldPass()" + }, + "metrics" : { + "step.endLine" : 5, + "step.startLine" : 5 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_3}, + "resource" : "* def pass = Java.type('org.example.Flaky').shouldPass()", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_6}, + "start" : ${content_start_6}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* match pass == true" + }, + "metrics" : { + "step.endLine" : 6, + "step.startLine" : 6 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* match pass == true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_7}, + "start" : ${content_start_7}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_8}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* match pass == true" + }, + "metrics" : { + "step.endLine" : 7, + "step.startLine" : 7 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* match pass == true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_8}, + "start" : ${content_start_8}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_9}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* match pass == true" + }, + "metrics" : { + "step.endLine" : 6, + "step.startLine" : 6 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_2}, + "resource" : "* match pass == true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_9}, + "start" : ${content_start_9}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_10}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* match pass == true" + }, + "metrics" : { + "step.endLine" : 7, + "step.startLine" : 7 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_2}, + "resource" : "* match pass == true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_10}, + "start" : ${content_start_10}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_11}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* match pass == true" + }, + "metrics" : { + "step.endLine" : 6, + "step.startLine" : 6 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_3}, + "resource" : "* match pass == true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_11}, + "start" : ${content_start_11}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_12}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* match pass == true" + }, + "metrics" : { + "step.endLine" : 7, + "step.startLine" : 7 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_3}, + "resource" : "* match pass == true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_12}, + "start" : ${content_start_12}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_13}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.status" : "pass", + "test.suite" : "[org/example/test_continue_on_step_failure] test continue on step failure", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "karate.test_suite", + "resource" : "[org/example/test_continue_on_step_failure] test continue on step failure", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_13}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_14}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack}, + "error.type" : "com.intuit.karate.KarateException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.failure_suppressed" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.name" : "flaky scenario", + "test.status" : "fail", + "test.suite" : "[org/example/test_continue_on_step_failure] test continue on step failure", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_4}, + "resource" : "[org/example/test_continue_on_step_failure] test continue on step failure.flaky scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id}, + "start" : ${content_start_14}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_15}, + "error" : 1, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "error.message" : ${content_meta_error_message}, + "error.stack" : ${content_meta_error_stack_2}, + "error.type" : "com.intuit.karate.KarateException", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.failure_suppressed" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_retry" : "true", + "test.module" : "karate-1.0", + "test.name" : "flaky scenario", + "test.retry_reason" : "auto_test_retry", + "test.status" : "fail", + "test.suite" : "[org/example/test_continue_on_step_failure] test continue on step failure", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_4}, + "resource" : "[org/example/test_continue_on_step_failure] test continue on step failure.flaky scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_2}, + "start" : ${content_start_15}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_16}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_retry" : "true", + "test.module" : "karate-1.0", + "test.name" : "flaky scenario", + "test.retry_reason" : "auto_test_retry", + "test.status" : "pass", + "test.suite" : "[org/example/test_continue_on_step_failure] test continue on step failure", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_4}, + "resource" : "[org/example/test_continue_on_step_failure] test continue on step failure.flaky scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_3}, + "start" : ${content_start_16}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_17}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "karate-1.0", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test_session", + "resource" : "karate-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_17}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_18}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6} + }, + "name" : "karate.test_module", + "resource" : "karate-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_18}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file From 9c0449a48e22e72cd9271940adfd0b3486af9ac4 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Mon, 29 Jun 2026 14:24:23 +0200 Subject: [PATCH 4/5] fix: spotless --- .../karate/KarateExecutionInstrumentation.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java index 6666445f5ca..2d455404983 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java @@ -117,8 +117,8 @@ public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { } // When the scenario was retried, the original runtime's result must reflect the final - // attempt's outcome. To avoid final field modifications, the final attempt's failure is reflected onto - // the original result via addStepResult + // attempt's outcome. To avoid final field modifications, the final attempt's failure is + // reflected onto the original result via addStepResult if (finalResult.isFailed() && !originalResult.isFailed()) { originalResult.addStepResult(finalResult.getFailedStep()); } From 3eaec0c7b34952065e9018f795fb09291c3ba9ed Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Mon, 29 Jun 2026 15:20:43 +0200 Subject: [PATCH 5/5] nit: comment --- .../instrumentation/karate/KarateExecutionInstrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java index 2d455404983..e218b5b60cc 100644 --- a/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java @@ -116,7 +116,7 @@ public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { finalResult = retry.result; } - // When the scenario was retried, the original runtime's result must reflect the final + // When the scenario is retried, the original runtime's result must reflect the final // attempt's outcome. To avoid final field modifications, the final attempt's failure is // reflected onto the original result via addStepResult if (finalResult.isFailed() && !originalResult.isFailed()) {