From 85cff2c841c038be694e1c6fd17e8e1a2d4f98a4 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 27 Apr 2026 00:38:49 +0200 Subject: [PATCH 01/10] feat(api): add off-CPU and wall-clock park hooks to ProfilingContextIntegration --- .../api/ProfilingContextIntegration.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java index 4accced983a..3e67cf48fa3 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java @@ -34,6 +34,36 @@ default int encodeResourceName(CharSequence constant) { return 0; } + /** Returns the current TSC tick count for the calling thread. */ + default long getCurrentTicks() { + return 0L; + } + + /** + * Emits a TaskBlock event covering a blocking interval on the current thread. + * + * @param startTicks TSC tick at block entry + * @param spanId the span ID active when blocking began + * @param rootSpanId the local root span ID active when blocking began + * @param blocker identity hash code of the blocking object, or 0 if none + * @param unblockingSpanId the span ID of the thread that unblocked this thread, or 0 if unknown + */ + default void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) {} + + /** + * Called when the current thread is about to enter {@code LockSupport.park*}. Native code can + * suppress wall-clock signals for the park interval and record the start tick for off-CPU + * analysis. + */ + default void parkEnter(long spanId, long rootSpanId) {} + + /** + * Called when the current thread has returned from {@code LockSupport.park*}. Clears the park + * state and may emit a TaskBlock JFR event. + */ + default void parkExit(long blocker, long unblockingSpanId) {} + String name(); final class NoOp implements ProfilingContextIntegration { From 65eab2754009c2961b0025f129822941a1d25045 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 27 Apr 2026 09:01:08 +0200 Subject: [PATCH 02/10] feat(profiling): wire TaskBlock, park, ticks, and encode to JavaProfiler --- .../profiling/ddprof/DatadogProfiler.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index b1e07b08c32..28f79d27f91 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -341,6 +341,18 @@ public void recordTraceRoot(long rootSpanId, String endpoint, String operation) } } + /** Monotonic tick count for TaskBlock and wall-clock off-CPU interval timing. */ + public long getCurrentTicks() { + return profiler.getCurrentTicks(); + } + + int encode(CharSequence constant) { + if (constant != null && profiler != null) { + return contextSetter.encode(constant.toString()); + } + return 0; + } + public int operationNameOffset() { return offsetOf(OPERATION); } @@ -454,4 +466,24 @@ void recordQueueTimeEvent( } } } + + void recordTaskBlockEvent( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + if (profiler != null) { + long endTicks = profiler.getCurrentTicks(); + profiler.recordTaskBlock(startTicks, endTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + } + + void parkEnter(long spanId, long rootSpanId) { + if (profiler != null) { + profiler.parkEnter(spanId, rootSpanId); + } + } + + void parkExit(long blocker, long unblockingSpanId) { + if (profiler != null) { + profiler.parkExit(blocker, unblockingSpanId); + } + } } From 3ccd0cd2b75d66740c3f86ec8907a6cc69148c69 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 27 Apr 2026 09:17:29 +0200 Subject: [PATCH 03/10] feat(profiling): delegate park/TaskBlock/ticks in DatadogProfilingIntegration --- .../ddprof/DatadogProfilingIntegration.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 00a0358d346..67af75a7afd 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -71,11 +71,53 @@ public void onDetach() { } } + @Override + public int encode(CharSequence constant) { + return DDPROF.encode(constant); + } + + @Override + public int encodeOperationName(CharSequence constant) { + if (SPAN_NAME_INDEX >= 0) { + return DDPROF.encode(constant); + } + return 0; + } + + @Override + public int encodeResourceName(CharSequence constant) { + if (RESOURCE_NAME_INDEX >= 0) { + return DDPROF.encode(constant); + } + return 0; + } + @Override public String name() { return "ddprof"; } + @Override + public long getCurrentTicks() { + return DDPROF.getCurrentTicks(); + } + + @Override + public void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + DDPROF.recordTaskBlockEvent(startTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + + @Override + public void parkEnter(long spanId, long rootSpanId) { + DDPROF.parkEnter(spanId, rootSpanId); + } + + @Override + public void parkExit(long blocker, long unblockingSpanId) { + DDPROF.parkExit(blocker, unblockingSpanId); + } + public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); From 88b1f2c183d07b50fe7822e845d876131204c6cd Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 27 Apr 2026 09:19:28 +0200 Subject: [PATCH 04/10] feat(profiling): add LockSupport.park and Object.wait instrumentation --- .../lock-support-profiling/build.gradle | 11 + .../LockSupportProfilingInstrumentation.java | 125 +++++++++++ ...ckSupportProfilingInstrumentationTest.java | 174 +++++++++++++++ .../object-wait-profiling/build.gradle | 16 ++ .../ObjectWaitProfilingInstrumentation.java | 135 ++++++++++++ ...bjectWaitProfilingInstrumentationTest.java | 204 ++++++++++++++++++ settings.gradle.kts | 2 + 7 files changed, 667 insertions(+) create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java create mode 100644 dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java create mode 100644 dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/build.gradle create mode 100644 dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java create mode 100644 dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle new file mode 100644 index 00000000000..9935e818b49 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle @@ -0,0 +1,11 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + coreJdk() + } +} + +dependencies { + testImplementation libs.bundles.junit5 +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java new file mode 100644 index 00000000000..55a313fc91d --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -0,0 +1,125 @@ +package datadog.trace.instrumentation.locksupport; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isStatic; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import java.util.concurrent.ConcurrentHashMap; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.matcher.ElementMatchers; + +/** + * Instruments {@link java.util.concurrent.locks.LockSupport#park} variants to emit {@code + * datadog.TaskBlock} JFR events. These events record the span, root-span, and duration of every + * blocking interval, enabling critical-path analysis across async handoffs. + * + *

Also instruments {@link java.util.concurrent.locks.LockSupport#unpark} to capture the span ID + * of the unblocking thread, which is then recorded in the TaskBlock event. + * + *

{@code parkEnter} runs even without an active span (span id 0) so the native wall-clock + * precheck can suppress {@code SIGVTALRM} for the whole park interval. TaskBlock JFR emission is + * gated by the profiler on duration and span context. + */ +@AutoService(InstrumenterModule.class) +public class LockSupportProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + public LockSupportProfilingInstrumentation() { + super("lock-support-profiling"); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.util.concurrent.locks.LockSupport"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(nameStartsWith("park")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$ParkAdvice"); + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(ElementMatchers.named("unpark")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$UnparkAdvice"); + } + + /** Holds shared state accessible from both {@link ParkAdvice} and {@link UnparkAdvice}. */ + public static final class State { + /** Maps target thread to the span ID of the thread that called {@code unpark()} on it. */ + public static final ConcurrentHashMap UNPARKING_SPAN = new ConcurrentHashMap<>(); + } + + public static final class ParkAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static long[] before(@Advice.Argument(value = 0, optional = true) Object blocker) { + ProfilingContextIntegration profiling = AgentTracer.get().getProfilingContext(); + if (profiling == null) { + return null; + } + // Always call parkEnter for signal suppression, even without an active span. + // spanId/rootSpanId = 0 when no active span — no JFR event will be emitted at exit for + // zero-span intervals (the native code filters those out by duration threshold and span + // check). + long spanId = 0L; + long rootSpanId = 0L; + AgentSpan span = AgentTracer.activeSpan(); + if (span != null && span.context() instanceof ProfilerContext) { + ProfilerContext ctx = (ProfilerContext) span.context(); + spanId = ctx.getSpanId(); + rootSpanId = ctx.getRootSpanId(); + } + profiling.parkEnter(spanId, rootSpanId); + long blockerHash = blocker != null ? System.identityHashCode(blocker) : 0L; + return new long[] {blockerHash, spanId, rootSpanId}; + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter long[] state) { + // Always drain the map entry before any early return. If we returned first, a stale + // unblocking-span ID placed by a prior unpark() would persist and be incorrectly + // attributed to the next TaskBlock event emitted on this thread. + Long unblockingSpanId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + if (state == null) { + return; + } + // state = [blockerHash, spanId, rootSpanId] — set in before(). + // parkExit() records the TaskBlock JFR event using the start tick saved by parkEnter(). + AgentTracer.get() + .getProfilingContext() + .parkExit(state[0], unblockingSpanId != null ? unblockingSpanId : 0L); + } + } + + public static final class UnparkAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before(@Advice.Argument(0) Thread thread) { + if (thread == null) { + return; + } + AgentSpan span = AgentTracer.activeSpan(); + if (span == null || !(span.context() instanceof ProfilerContext)) { + return; + } + ProfilerContext ctx = (ProfilerContext) span.context(); + long effectiveSpanId = ctx.getSpanId(); + State.UNPARKING_SPAN.put(thread, effectiveSpanId); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java new file mode 100644 index 00000000000..1855a21744b --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java @@ -0,0 +1,174 @@ +package datadog.trace.instrumentation.locksupport; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.instrumentation.locksupport.LockSupportProfilingInstrumentation.State; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LockSupportProfilingInstrumentation}. + * + *

These tests exercise the {@link State} map directly, verifying the mechanism used to + * communicate the unblocking span ID from {@code UnparkAdvice} to {@code ParkAdvice}. + */ +class LockSupportProfilingInstrumentationTest { + + @BeforeEach + void clearState() { + State.UNPARKING_SPAN.clear(); + } + + @AfterEach + void cleanupState() { + State.UNPARKING_SPAN.clear(); + } + + // ------------------------------------------------------------------------- + // State map — basic contract + // ------------------------------------------------------------------------- + + @Test + void state_put_and_remove() { + Thread t = Thread.currentThread(); + long spanId = 12345L; + + State.UNPARKING_SPAN.put(t, spanId); + Long retrieved = State.UNPARKING_SPAN.remove(t); + + assertNotNull(retrieved); + assertEquals(spanId, (long) retrieved); + // After removal the entry should be gone + assertNull(State.UNPARKING_SPAN.get(t)); + } + + @Test + void state_remove_returns_null_when_absent() { + Thread t = new Thread(() -> {}); + assertNull(State.UNPARKING_SPAN.remove(t)); + } + + @Test + void state_is_initially_empty() { + assertTrue(State.UNPARKING_SPAN.isEmpty()); + } + + // ------------------------------------------------------------------------- + // Multithreaded: unpark thread populates map, parked thread reads it + // ------------------------------------------------------------------------- + + /** + * Simulates the UnparkAdvice → ParkAdvice handoff: + * + *

    + *
  1. Thread A (the "parked" thread) blocks on a latch. + *
  2. Thread B (the "unparking" thread) places its span ID in {@code State.UNPARKING_SPAN} for + * Thread A and then releases the latch. + *
  3. Thread A wakes up, reads and removes the span ID from the map. + *
+ */ + @Test + void unparking_spanId_is_visible_to_parked_thread() throws InterruptedException { + long unparkingSpanId = 99887766L; + + CountDownLatch ready = new CountDownLatch(1); + CountDownLatch go = new CountDownLatch(1); + AtomicLong capturedSpanId = new AtomicLong(-1L); + AtomicReference parkedThreadRef = new AtomicReference<>(); + + Thread parkedThread = + new Thread( + () -> { + parkedThreadRef.set(Thread.currentThread()); + ready.countDown(); + try { + go.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Simulate what ParkAdvice.after does: read and remove unblocking span id + Long unblockingId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + + parkedThread.start(); + ready.await(); // wait for parked thread to register itself + + // Simulate what UnparkAdvice.before does: record unparking span id + State.UNPARKING_SPAN.put(parkedThread, unparkingSpanId); + go.countDown(); // unblock parked thread + + parkedThread.join(2_000); + assertFalse(parkedThread.isAlive(), "Test thread did not finish in time"); + assertEquals( + unparkingSpanId, + capturedSpanId.get(), + "Parked thread should have read the unblocking span id placed by unparking thread"); + } + + /** + * Verifies that if no entry exists for the parked thread (i.e. the thread was unblocked by a + * non-traced thread), the {@code remove} returns {@code null} and the code falls back to 0. + */ + @Test + void no_unparking_entry_yields_zero() throws InterruptedException { + AtomicLong capturedSpanId = new AtomicLong(-1L); + + Thread parkedThread = + new Thread( + () -> { + Long unblockingId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + parkedThread.start(); + parkedThread.join(2_000); + + assertEquals( + 0L, capturedSpanId.get(), "Should fall back to 0 when no unparking span id is recorded"); + } + + // ------------------------------------------------------------------------- + // ParkAdvice.after — null state is a no-op + // ------------------------------------------------------------------------- + + /** + * When {@code ParkAdvice.before} returns {@code null} (profiler not active or no active span), + * {@code ParkAdvice.after} must not throw and must not leave entries in {@code UNPARKING_SPAN}. + * It does call {@code remove(currentThread)}, but on an empty map that is a no-op. + */ + @Test + void parkAdvice_after_null_state_isNoOp() { + LockSupportProfilingInstrumentation.ParkAdvice.after(null); + assertTrue(State.UNPARKING_SPAN.isEmpty()); + } + + /** + * Regression test for stale-entry misattribution. + * + *

If {@code unpark(t)} is called (inserting an entry into {@code UNPARKING_SPAN}) and thread + * {@code t} then parks without an active span ({@code state == null}), the entry must still be + * drained. Without the fix, it would linger and be incorrectly attributed to the next {@code + * TaskBlock} emitted on that thread. + */ + @Test + void stale_entry_is_drained_when_park_fires_without_active_span() { + Thread t = Thread.currentThread(); + State.UNPARKING_SPAN.put(t, 99L); + + // Simulate park() returning with no active span (state == null) + LockSupportProfilingInstrumentation.ParkAdvice.after(null); + + assertNull( + State.UNPARKING_SPAN.get(t), + "Stale UNPARKING_SPAN entry must be drained even when state is null"); + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/build.gradle new file mode 100644 index 00000000000..d61e8bd6155 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/build.gradle @@ -0,0 +1,16 @@ +apply from: "$rootDir/gradle/java.gradle" + +testJvmConstraints { + minJavaVersion = JavaVersion.VERSION_21 +} + +muzzle { + pass { + coreJdk('21') + } +} + +dependencies { + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java new file mode 100644 index 00000000000..5a8d7db2d21 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java @@ -0,0 +1,135 @@ +package datadog.trace.instrumentation.objectwait; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import datadog.environment.JavaVirtualMachine; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import net.bytebuddy.asm.Advice; + +/** + * Instruments {@link Object#wait(long)} in JDK 21+ to emit {@code datadog.TaskBlock} JFR events. + * + *

In JDK 21+, {@code wait(long)} is a pure-Java wrapper around the native {@code wait0(long)}, + * so ByteBuddy can add advice to it. In JDK 8-20 the method is declared {@code native} and is not + * instrumented by this class (Approach 1 osThreadState precheck already suppresses SIGVTALRM for + * threads in OBJECT_WAIT state on all JDK versions). + * + *

Only {@code wait(long)} is instrumented: {@code wait()} delegates to {@code wait(0L)} and + * {@code wait(long, int)} delegates to {@code wait(long)}, so all wait variants are covered. + * + *

{@code unblockingSpanId} is always 0 because {@code notify()} and {@code notifyAll()} remain + * {@code native} in JDK 21+ and the notifying thread cannot be identified via BCI. + */ +@AutoService(InstrumenterModule.class) +public class ObjectWaitProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + public ObjectWaitProfilingInstrumentation() { + super("object-wait-profiling"); + } + + @Override + public boolean isEnabled() { + return JavaVirtualMachine.isJavaVersionAtLeast(21) && super.isEnabled(); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.lang.Object"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(named("wait")) + .and(takesArguments(1)) + .and(takesArgument(0, long.class)) + .and(isDeclaredBy(named("java.lang.Object"))), + getClass().getName() + "$WaitAdvice"); + } + + public static final class WaitAdvice { + + // 1 ms — matches the default LockSupport park threshold in parkExit0 + static final long MIN_WAIT_NANOS = 1_000_000L; + + // State array indices — package-private for readability in tests + static final int IDX_BLOCKER = 0; + static final int IDX_SPAN_ID = 1; + static final int IDX_ROOT_SPAN_ID = 2; + static final int IDX_START_TICKS = 3; + static final int IDX_START_NANOS = 4; + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static long[] before(@Advice.This Object monitor) { + return captureState( + monitor, AgentTracer.get().getProfilingContext(), AgentTracer.activeSpan()); + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter long[] state) { + if (state == null) { + return; + } + ProfilingContextIntegration profiling = AgentTracer.get().getProfilingContext(); + if (profiling == null) { + return; + } + emitIfLongEnough(state, profiling); + } + + /** + * Captures wait-entry state. Package-private to allow direct unit-testing without a live agent. + * + * @return a 5-element {@code long[]} on success, or {@code null} when the preconditions are not + * met (no profiling context, no active span, or span context is not a {@link + * ProfilerContext}) + */ + static long[] captureState( + Object monitor, ProfilingContextIntegration profiling, AgentSpan span) { + if (profiling == null) { + return null; + } + if (span == null || !(span.context() instanceof ProfilerContext)) { + return null; + } + ProfilerContext ctx = (ProfilerContext) span.context(); + return new long[] { + System.identityHashCode(monitor), // [IDX_BLOCKER] monitor identity + ctx.getSpanId(), // [IDX_SPAN_ID] + ctx.getRootSpanId(), // [IDX_ROOT_SPAN_ID] + profiling.getCurrentTicks(), // [IDX_START_TICKS] TSC ticks for event timing + System.nanoTime() // [IDX_START_NANOS] wall-clock nanos for duration filter + }; + } + + /** + * Emits a TaskBlock event if the elapsed wall time since entry exceeds {@link #MIN_WAIT_NANOS}. + * Package-private to allow direct unit-testing without a live agent. + */ + static void emitIfLongEnough(long[] state, ProfilingContextIntegration profiling) { + if (System.nanoTime() - state[IDX_START_NANOS] < MIN_WAIT_NANOS) { + return; + } + // unblockingSpanId = 0: notify/notifyAll are native in JDK 21+, + // so the notifying thread cannot be identified via BCI. + profiling.recordTaskBlock( + state[IDX_START_TICKS], + state[IDX_SPAN_ID], + state[IDX_ROOT_SPAN_ID], + state[IDX_BLOCKER], + 0L); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java new file mode 100644 index 00000000000..e1e4f158fa2 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java @@ -0,0 +1,204 @@ +package datadog.trace.instrumentation.objectwait; + +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_BLOCKER; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_ROOT_SPAN_ID; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_SPAN_ID; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_START_NANOS; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_START_TICKS; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.MIN_WAIT_NANOS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +// Combining both interfaces as span.context() return type is AgentSpanContext, +// but production code casts it to ProfilerContext — both must be satisfied by the mock. +// AgentSpanContext is the declared return type; ProfilerContext provides getSpanId/getRootSpanId. + +/** + * Unit tests for {@link ObjectWaitProfilingInstrumentation}. + * + *

These tests exercise the two package-private helpers ({@code captureState} and {@code + * emitIfLongEnough}) directly, bypassing the {@code AgentTracer.get()} call that would require a + * live agent. The {@link WaitAdvice#after(long[])} null-guard is tested separately. + */ +@ExtendWith(MockitoExtension.class) +class ObjectWaitProfilingInstrumentationTest { + + private static final long SPAN_ID = 0xDEADBEEFL; + private static final long ROOT_SPAN_ID = 0xCAFEBABEL; + private static final long START_TICKS = 42_000_000L; + + /** + * Implements both interfaces: span.context() returns AgentSpanContext but is cast to + * ProfilerContext. + */ + private interface ProfilerSpanContext extends AgentSpanContext, ProfilerContext {} + + @Mock private ProfilingContextIntegration profiling; + @Mock private AgentSpan span; + @Mock private ProfilerSpanContext ctx; + @Mock private AgentSpanContext nonProfilerCtx; + + private final Object monitor = new Object(); + + @BeforeEach + void setUp() { + lenient().when(span.context()).thenReturn(ctx); + lenient().when(ctx.getSpanId()).thenReturn(SPAN_ID); + lenient().when(ctx.getRootSpanId()).thenReturn(ROOT_SPAN_ID); + lenient().when(profiling.getCurrentTicks()).thenReturn(START_TICKS); + } + + // --------------------------------------------------------------------------- + // WaitAdvice.after — null state guard + // --------------------------------------------------------------------------- + + @Test + void after_nullState_doesNotThrow() { + // before() returns null when profiling is absent or no active span; + // after() must be a safe no-op in that case. + WaitAdvice.after(null); + } + + // --------------------------------------------------------------------------- + // captureState — precondition gates + // --------------------------------------------------------------------------- + + @Test + void captureState_nullProfiling_returnsNull() { + assertNull(WaitAdvice.captureState(monitor, null, span)); + } + + @Test + void captureState_nullSpan_returnsNull() { + assertNull(WaitAdvice.captureState(monitor, profiling, null)); + } + + @Test + void captureState_spanContextNotProfilerContext_returnsNull() { + lenient().when(span.context()).thenReturn(nonProfilerCtx); + assertNull(WaitAdvice.captureState(monitor, profiling, span)); + } + + // --------------------------------------------------------------------------- + // captureState — happy path + // --------------------------------------------------------------------------- + + @Test + void captureState_validSpan_returnsCorrectState() { + long before = System.nanoTime(); + long[] state = WaitAdvice.captureState(monitor, profiling, span); + long after = System.nanoTime(); + + assertNotNull(state); + assertEquals(5, state.length); + assertEquals(System.identityHashCode(monitor), state[IDX_BLOCKER]); + assertEquals(SPAN_ID, state[IDX_SPAN_ID]); + assertEquals(ROOT_SPAN_ID, state[IDX_ROOT_SPAN_ID]); + assertEquals(START_TICKS, state[IDX_START_TICKS]); + // startNanos must be within the window captured around the call + org.junit.jupiter.api.Assertions.assertTrue( + state[IDX_START_NANOS] >= before && state[IDX_START_NANOS] <= after, + "startNanos should be captured during captureState()"); + } + + @Test + void captureState_differentMonitors_produceDifferentBlockerHashes() { + Object monitorA = new Object(); + Object monitorB = new Object(); + + long[] stateA = WaitAdvice.captureState(monitorA, profiling, span); + long[] stateB = WaitAdvice.captureState(monitorB, profiling, span); + + assertNotNull(stateA); + assertNotNull(stateB); + // Different object identities must produce different blocker hashes + // (identity hash codes can collide, but for freshly allocated objects this holds + // in all current JVM implementations — and what matters is they use identityHashCode) + assertEquals(System.identityHashCode(monitorA), stateA[IDX_BLOCKER]); + assertEquals(System.identityHashCode(monitorB), stateB[IDX_BLOCKER]); + } + + // --------------------------------------------------------------------------- + // emitIfLongEnough — duration filter + // --------------------------------------------------------------------------- + + @Test + void emitIfLongEnough_shortWait_doesNotEmit() { + // Set startNanos to "just now" so elapsed < MIN_WAIT_NANOS + long[] state = buildState(System.nanoTime()); + + WaitAdvice.emitIfLongEnough(state, profiling); + + verifyNoInteractions(profiling); + } + + @Test + void emitIfLongEnough_longWait_emitsTaskBlock() { + // Set startNanos to 2 ms ago — comfortably above the 1 ms threshold + long twoMsAgo = System.nanoTime() - 2 * MIN_WAIT_NANOS; + long[] state = buildState(twoMsAgo); + + WaitAdvice.emitIfLongEnough(state, profiling); + + verify(profiling) + .recordTaskBlock( + START_TICKS, // startTicks + SPAN_ID, // spanId + ROOT_SPAN_ID, // rootSpanId + System.identityHashCode(monitor), // blocker + 0L); // unblockingSpanId always 0 (notify is native) + } + + @Test + void emitIfLongEnough_exactlyAtThreshold_emitsTaskBlock() { + // nanos exactly at the boundary — elapsed == MIN_WAIT_NANOS is NOT < threshold, so it emits + long atThreshold = System.nanoTime() - MIN_WAIT_NANOS; + long[] state = buildState(atThreshold); + + WaitAdvice.emitIfLongEnough(state, profiling); + + verify(profiling) + .recordTaskBlock(START_TICKS, SPAN_ID, ROOT_SPAN_ID, System.identityHashCode(monitor), 0L); + } + + @Test + void emitIfLongEnough_unblockingSpanIdIsAlwaysZero() { + long twoMsAgo = System.nanoTime() - 2 * MIN_WAIT_NANOS; + long[] state = buildState(twoMsAgo); + + WaitAdvice.emitIfLongEnough(state, profiling); + + // The 5th argument (unblockingSpanId) must always be 0 because notify/notifyAll + // are native in JDK 21+ — the notifying thread cannot be identified via BCI. + verify(profiling).recordTaskBlock(START_TICKS, SPAN_ID, ROOT_SPAN_ID, state[IDX_BLOCKER], 0L); + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + private long[] buildState(long startNanos) { + return new long[] { + System.identityHashCode(monitor), // IDX_BLOCKER + SPAN_ID, // IDX_SPAN_ID + ROOT_SPAN_ID, // IDX_ROOT_SPAN_ID + START_TICKS, // IDX_START_TICKS + startNanos // IDX_START_NANOS + }; + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 6d3d15d16f1..697c7f9e2f9 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -328,6 +328,8 @@ include( ":dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin", ":dd-java-agent:instrumentation:datadog:profiling:enable-wallclock-profiling", ":dd-java-agent:instrumentation:datadog:profiling:exception-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:lock-support-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:object-wait-profiling", ":dd-java-agent:instrumentation:datadog:tracing:trace-annotation", ":dd-java-agent:instrumentation:datanucleus-4.0.5", ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0", From 9701ba63b0a408be5fc3abba0e370268824f01ae Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 27 Apr 2026 16:35:17 +0200 Subject: [PATCH 05/10] fix --- .../profiling/ddprof/DatadogProfiler.java | 5 +-- .../profiling/ddprof/DatadogProfilerTest.java | 44 +++++++++++++++++++ ...ckSupportProfilingInstrumentationTest.java | 16 +++++++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index 28f79d27f91..8abf48b769b 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -347,9 +347,8 @@ public long getCurrentTicks() { } int encode(CharSequence constant) { - if (constant != null && profiler != null) { - return contextSetter.encode(constant.toString()); - } + // java-profiler ContextSetter no longer exposes value encoding. + // Keep API contract by returning "not encoded" (0), which callers already handle. return 0; } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index 55d39ba52a0..80754c7937f 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.Properties; import java.util.UUID; +import java.util.concurrent.locks.LockSupport; import java.util.stream.IntStream; import java.util.stream.Stream; import org.junit.jupiter.api.Assumptions; @@ -29,6 +30,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.ItemFilters; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,6 +77,47 @@ void test() throws Exception { } } + @Test + void testTaskBlockBridgeMethodsEmitTaskBlockEvents() throws Exception { + assertDoesNotThrow( + () -> DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(), "Profiler not available"); + DatadogProfiler profiler = DatadogProfiler.newInstance(ConfigProvider.getInstance()); + if (profiler.isActive()) { + log.warn("Datadog profiler is already running. Skipping task-block integration test."); + return; + } + + OngoingRecording recording = profiler.start(); + if (recording == null) { + log.warn("Datadog Profiler is not available. Skipping task-block integration test."); + return; + } + + try { + // Direct bridge path (recordTaskBlock -> JavaProfiler.recordTaskBlock0) + long startTicks = profiler.getCurrentTicks(); + LockSupport.parkNanos(3_000_000L); // > 1ms native threshold + profiler.recordTaskBlockEvent(startTicks, 101L, 202L, 303L, 404L); + + // Park path (parkEnter/parkExit -> JavaProfiler.parkEnter0/parkExit0) + profiler.parkEnter(505L, 606L); + LockSupport.parkNanos(3_000_000L); // > 1ms native threshold + profiler.parkExit(707L, 808L); + + RecordingData data = profiler.stop(recording); + assertNotNull(data); + IItemCollection events = JfrLoaderToolkit.loadEvents(data.getStream()); + long taskBlockCount = + events.apply(ItemFilters.type("datadog.TaskBlock")).stream() + .mapToLong(IItemIterable::getItemCount) + .sum(); + + assertTrue(taskBlockCount > 0, "Expected datadog.TaskBlock events from bridge methods"); + } finally { + recording.stop(); + } + } + @ParameterizedTest @MethodSource("profilingModes") void testStartCmd(boolean cpu, boolean wall, boolean alloc, boolean memleak) throws Exception { diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java index 1855a21744b..c2a16b13868 100644 --- a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java @@ -171,4 +171,20 @@ void stale_entry_is_drained_when_park_fires_without_active_span() { State.UNPARKING_SPAN.get(t), "Stale UNPARKING_SPAN entry must be drained even when state is null"); } + + /** + * If multiple unpark calls race for the same parked thread, the latest span ID should be consumed + * and the entry must still be drained exactly once by ParkAdvice.after(). + */ + @Test + void latest_unparking_span_wins_and_entry_is_drained() { + Thread t = Thread.currentThread(); + State.UNPARKING_SPAN.put(t, 101L); + State.UNPARKING_SPAN.put(t, 202L); + + Long consumed = State.UNPARKING_SPAN.remove(t); + assertNotNull(consumed); + assertEquals(202L, consumed.longValue()); + assertNull(State.UNPARKING_SPAN.get(t), "Entry must be removed after consumption"); + } } From 769b1690fc05fb808a07dd137170f8f6d24b6377 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sun, 3 May 2026 17:14:00 +0200 Subject: [PATCH 06/10] test --- .gitlab-ci.yml | 104 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab93e6d2171..fa0a7e8afb9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ stages: - java-spring-petclinic-parallel-slo - java-startup-parallel - java-load-parallel + - java-load-parallel-slo - java-dacapo-parallel - shared-pipeline - benchmarks @@ -62,6 +63,10 @@ variables: description: "Enable flaky tests" value: "false" + JAVA_PROFILER_REF: + description: "When non-empty, clone DataDog/java-profiler at this Git ref (branch or tag), build ddprof, and use it as ddprof.jar for Gradle jobs instead of the Maven dependency." + value: "paul.fournillon/wallclock_precheck" + # One pipeline injection package size ratchet OCI_PACKAGE_MAX_SIZE_BYTES: 40_000_000 LIB_INJECTION_IMAGE_MAX_SIZE_BYTES: 40_000_000 @@ -165,9 +170,21 @@ default: echo "Failed to find base ref for PR" >&2 fi +# When build_java_profiler_ddprof ran, its artifact is available at custom-ddprof/ddprof.jar. +# Append root project property expected by dd-java-agent/ddprof-lib/build.gradle. +.inject_custom_ddprof_jar: &inject_custom_ddprof_jar + - | + if [ -f "${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" ]; then + echo "ddprof.jar=${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" >> gradle.properties + echo "Using custom ddprof.jar from java-profiler build" + fi + .gradle_build: &gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build + needs: + - job: build_java_profiler_ddprof + optional: true variables: MAVEN_OPTS: "-Xms256M -Xmx1024M" GRADLE_WORKERS: 6 @@ -217,6 +234,7 @@ default: org.gradle.java.installations.auto-download=false org.gradle.java.installations.fromEnv=$JAVA_HOMES EOF + - *inject_custom_ddprof_jar - mkdir -p .gradle - export GRADLE_USER_HOME=$(pwd)/.gradle # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties @@ -292,8 +310,43 @@ dd-octo-sts-pre-release-check: max: 2 when: always +# Builds java-profiler from JAVA_PROFILER_REF and publishes custom-ddprof/ddprof.jar for downstream Gradle jobs. +# Requires JDK 17+ for Gradle 9 and native toolchain for ddprof release (same base image as .gradle_build). +build_java_profiler_ddprof: + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base + stage: build + rules: + - if: '$JAVA_PROFILER_REF =~ /.+/' + when: on_success + variables: + FF_USE_FASTZIP: "true" + CACHE_COMPRESSION_LEVEL: "slowest" + KUBERNETES_CPU_REQUEST: 10 + KUBERNETES_MEMORY_REQUEST: 20Gi + KUBERNETES_MEMORY_LIMIT: 20Gi + script: + - | + set -euo pipefail + mkdir -p "${CI_PROJECT_DIR}/custom-ddprof" + SRCDIR="${CI_PROJECT_DIR}/java-profiler-src" + rm -rf "$SRCDIR" + git clone --depth 1 --branch "$JAVA_PROFILER_REF" https://github.com/DataDog/java-profiler.git "$SRCDIR" + cd "$SRCDIR" + chmod +x ./gradlew + ./gradlew --version + ./gradlew assembleRelease -Pskip-tests -Pskip-gtest + JAR=$(ls ddprof-lib/build/libs/ddprof-*.jar | grep -v -- '-sources' | head -1) + cp "$JAR" "${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" + ls -la "${CI_PROJECT_DIR}/custom-ddprof/" + artifacts: + when: on_success + paths: + - custom-ddprof/ddprof.jar + build: needs: + - job: build_java_profiler_ddprof + optional: true - job: maven-central-pre-release-check optional: true - job: dd-octo-sts-pre-release-check @@ -404,7 +457,9 @@ publish-artifacts-to-s3: spotless: extends: .gradle_build stage: tests - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true variables: GRADLE_MEMORY_MAX: 6G script: @@ -414,7 +469,9 @@ spotless: check-instrumentation-naming: extends: .gradle_build stage: tests - needs: [ ] + needs: + - job: build_java_profiler_ddprof + optional: true script: - ./gradlew --version - ./gradlew checkInstrumentationNaming @@ -422,7 +479,9 @@ check-instrumentation-naming: config-inversion-linter: extends: .gradle_build stage: tests - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true script: - ./gradlew --version - ./gradlew checkConfigurations @@ -431,7 +490,10 @@ test_published_artifacts: extends: .gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests stage: tests - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" script: @@ -458,7 +520,10 @@ test_published_artifacts: .check_job: extends: .gradle_build - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build stage: tests variables: CACHE_TYPE: "lib" @@ -494,7 +559,9 @@ test_published_artifacts: check_build_src: extends: .check_job - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true variables: GRADLE_TARGET: ":buildSrc:build" @@ -529,7 +596,10 @@ check_debugger: muzzle: extends: .gradle_build - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests parallel: matrix: @@ -561,7 +631,10 @@ muzzle: muzzle-dep-report: extends: .gradle_build - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests variables: CACHE_TYPE: "inst" @@ -598,7 +671,10 @@ muzzle-dep-report: extends: .gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests variables: GRADLE_PARAMS: "-PskipFlakyTests" @@ -896,7 +972,10 @@ deploy_to_di_backend:manual: deploy_to_maven_central: extends: .gradle_build stage: publish - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" rules: @@ -924,7 +1003,10 @@ deploy_to_maven_central: deploy_snapshot_with_ddprof_snapshot: extends: .gradle_build stage: publish - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" rules: From e2f945121b3d4f1f0d29864113793ba9915c7db0 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sun, 3 May 2026 19:23:29 +0200 Subject: [PATCH 07/10] fix test --- .gitlab-ci.yml | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3f1f4cd01f..efbaab125d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -312,7 +312,7 @@ dd-octo-sts-pre-release-check: when: always # Builds java-profiler from JAVA_PROFILER_REF and publishes custom-ddprof/ddprof.jar for downstream Gradle jobs. -# Requires JDK 17+ for Gradle 9 and native toolchain for ddprof release (same base image as .gradle_build). +# Uses :ddprof-lib:assembleReleaseJar (not assembleRelease, which is native-only). JDK 21+ for release + JDK 17+ for Gradle 9. build_java_profiler_ddprof: image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build @@ -325,6 +325,30 @@ build_java_profiler_ddprof: KUBERNETES_CPU_REQUEST: 10 KUBERNETES_MEMORY_REQUEST: 20Gi KUBERNETES_MEMORY_LIMIT: 20Gi + before_script: + - | + # java-profiler uses Gradle 9.x; Gradle requires JVM 17+. Builder image default java is often JDK 8. + if [ -n "${JAVA_21_HOME:-}" ] && [ -x "${JAVA_21_HOME}/bin/java" ]; then + export JAVA_HOME="$JAVA_21_HOME" + elif [ -n "${JAVA_17_HOME:-}" ] && [ -x "${JAVA_17_HOME}/bin/java" ]; then + export JAVA_HOME="$JAVA_17_HOME" + else + shopt -s nullglob + for d in /usr/lib/jvm/java-21-* /usr/lib/jvm/temurin-21-* /usr/lib/jvm/java-17-*; do + if [ -x "${d}/bin/java" ]; then + export JAVA_HOME="$d" + break + fi + done + shopt -u nullglob + fi + if [ -z "${JAVA_HOME:-}" ] || ! [ -x "${JAVA_HOME}/bin/java" ]; then + echo "Could not find JDK 17+ for Gradle 9 (set JAVA_21_HOME or JAVA_17_HOME, or install JDK 21 under /usr/lib/jvm)." >&2 + ls -la /usr/lib/jvm 2>/dev/null || true + exit 1 + fi + export PATH="${JAVA_HOME}/bin:${PATH}" + java -version script: - | set -euo pipefail @@ -335,8 +359,14 @@ build_java_profiler_ddprof: cd "$SRCDIR" chmod +x ./gradlew ./gradlew --version - ./gradlew assembleRelease -Pskip-tests -Pskip-gtest - JAR=$(ls ddprof-lib/build/libs/ddprof-*.jar | grep -v -- '-sources' | head -1) + # assembleRelease is the native link/assemble task only; the packaged jar is assembleReleaseJar. + ./gradlew :ddprof-lib:assembleReleaseJar -Pskip-tests -Pskip-gtest + JAR=$(find ddprof-lib/build/libs -maxdepth 1 -type f \( -name 'ddprof-*.jar' \) ! -name '*-sources*' ! -name '*-javadoc*' | head -1) + if [ -z "$JAR" ] || [ ! -f "$JAR" ]; then + echo "No ddprof jar found under ddprof-lib/build/libs" >&2 + ls -la ddprof-lib/build/libs 2>/dev/null || ls -laR ddprof-lib/build 2>/dev/null || true + exit 1 + fi cp "$JAR" "${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" ls -la "${CI_PROJECT_DIR}/custom-ddprof/" artifacts: From eba5e29dfcc69d8015b75d62fb1becf1dc3d633b Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Sun, 3 May 2026 22:50:04 +0200 Subject: [PATCH 08/10] fix(profiling): fix intrumentation --- .../locksupport/LockSupportProfilingInstrumentation.java | 6 ++++++ .../objectwait/ObjectWaitProfilingInstrumentation.java | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java index 55a313fc91d..30507bd50ab 100644 --- a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -42,6 +42,12 @@ public String[] knownMatchingTypes() { return new String[] {"java.util.concurrent.locks.LockSupport"}; } + @Override + public String[] muzzleIgnoredClassNames() { + // Advice references this nested holder; it lives on the instrumentation classpath only. + return new String[] {getClass().getName() + "$State"}; + } + @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java index 5a8d7db2d21..47fa356e62b 100644 --- a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java @@ -48,6 +48,13 @@ public String[] knownMatchingTypes() { return new String[] {"java.lang.Object"}; } + @Override + public String[] muzzleIgnoredClassNames() { + // Static helpers on the advice class produce intra-class references that core-JDK muzzle + // cannot resolve against an empty application classpath. + return new String[] {getClass().getName() + "$WaitAdvice"}; + } + @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( From ec177c97d0d183c9a8dfa4feee0d040f17eb7f94 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 4 May 2026 09:34:11 +0200 Subject: [PATCH 09/10] fix(profiling): add lock-support and object-wait en var to supported configurations --- metadata/supported-configurations.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index d77b600b2f6..6b3ae195243 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7865,6 +7865,14 @@ "aliases": ["DD_TRACE_INTEGRATION_LIBERTY_ENABLED", "DD_INTEGRATION_LIBERTY_ENABLED"] } ], + "DD_TRACE_LOCK_SUPPORT_PROFILING_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_LOCK_SUPPORT_PROFILING_ENABLED", "DD_INTEGRATION_LOCK_SUPPORT_PROFILING_ENABLED"] + } + ], "DD_TRACE_LOG4J_1_ENABLED": [ { "version": "A", @@ -8289,6 +8297,14 @@ "aliases": ["DD_OBFUSCATION_QUERY_STRING_REGEXP"] } ], + "DD_TRACE_OBJECT_WAIT_PROFILING_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_OBJECT_WAIT_PROFILING_ENABLED", "DD_INTEGRATION_OBJECT_WAIT_PROFILING_ENABLED"] + } + ], "DD_TRACE_OGNL_ENABLED": [ { "version": "A", From 0084005f1b88f5bc8996be9dfa330f736e3d5410 Mon Sep 17 00:00:00 2001 From: Paul Fournillon Date: Mon, 4 May 2026 14:04:37 +0200 Subject: [PATCH 10/10] chore: rename lock-support-profiling in lock-support and object-wait-profiling in object-wait --- .../{lock-support-profiling => lock-support}/build.gradle | 0 .../locksupport/LockSupportProfilingInstrumentation.java | 2 +- .../LockSupportProfilingInstrumentationTest.java | 0 .../{object-wait-profiling => object-wait}/build.gradle | 0 .../objectwait/ObjectWaitProfilingInstrumentation.java | 2 +- .../ObjectWaitProfilingInstrumentationTest.java | 0 metadata/supported-configurations.json | 8 ++++---- settings.gradle.kts | 4 ++-- 8 files changed, 8 insertions(+), 8 deletions(-) rename dd-java-agent/instrumentation/datadog/profiling/{lock-support-profiling => lock-support}/build.gradle (100%) rename dd-java-agent/instrumentation/datadog/profiling/{lock-support-profiling => lock-support}/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java (99%) rename dd-java-agent/instrumentation/datadog/profiling/{lock-support-profiling => lock-support}/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java (100%) rename dd-java-agent/instrumentation/datadog/profiling/{object-wait-profiling => object-wait}/build.gradle (100%) rename dd-java-agent/instrumentation/datadog/profiling/{object-wait-profiling => object-wait}/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java (99%) rename dd-java-agent/instrumentation/datadog/profiling/{object-wait-profiling => object-wait}/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java (100%) diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle similarity index 100% rename from dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/build.gradle rename to dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java similarity index 99% rename from dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java rename to dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java index 30507bd50ab..c6607ed5f16 100644 --- a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -34,7 +34,7 @@ public class LockSupportProfilingInstrumentation extends InstrumenterModule.Prof implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { public LockSupportProfilingInstrumentation() { - super("lock-support-profiling"); + super("lock-support"); } @Override diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java similarity index 100% rename from dd-java-agent/instrumentation/datadog/profiling/lock-support-profiling/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java rename to dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle similarity index 100% rename from dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/build.gradle rename to dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java similarity index 99% rename from dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java rename to dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java index 47fa356e62b..a96357fe4a9 100644 --- a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java @@ -35,7 +35,7 @@ public class ObjectWaitProfilingInstrumentation extends InstrumenterModule.Profi implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { public ObjectWaitProfilingInstrumentation() { - super("object-wait-profiling"); + super("object-wait"); } @Override diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java similarity index 100% rename from dd-java-agent/instrumentation/datadog/profiling/object-wait-profiling/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java rename to dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 6b3ae195243..7ac7866be94 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7865,12 +7865,12 @@ "aliases": ["DD_TRACE_INTEGRATION_LIBERTY_ENABLED", "DD_INTEGRATION_LIBERTY_ENABLED"] } ], - "DD_TRACE_LOCK_SUPPORT_PROFILING_ENABLED": [ + "DD_TRACE_LOCK_SUPPORT_ENABLED": [ { "version": "A", "type": "boolean", "default": "true", - "aliases": ["DD_TRACE_INTEGRATION_LOCK_SUPPORT_PROFILING_ENABLED", "DD_INTEGRATION_LOCK_SUPPORT_PROFILING_ENABLED"] + "aliases": ["DD_TRACE_INTEGRATION_LOCK_SUPPORT_ENABLED", "DD_INTEGRATION_LOCK_SUPPORT_ENABLED"] } ], "DD_TRACE_LOG4J_1_ENABLED": [ @@ -8297,12 +8297,12 @@ "aliases": ["DD_OBFUSCATION_QUERY_STRING_REGEXP"] } ], - "DD_TRACE_OBJECT_WAIT_PROFILING_ENABLED": [ + "DD_TRACE_OBJECT_WAIT_ENABLED": [ { "version": "A", "type": "boolean", "default": "true", - "aliases": ["DD_TRACE_INTEGRATION_OBJECT_WAIT_PROFILING_ENABLED", "DD_INTEGRATION_OBJECT_WAIT_PROFILING_ENABLED"] + "aliases": ["DD_TRACE_INTEGRATION_OBJECT_WAIT_ENABLED", "DD_INTEGRATION_OBJECT_WAIT_ENABLED"] } ], "DD_TRACE_OGNL_ENABLED": [ diff --git a/settings.gradle.kts b/settings.gradle.kts index 67e5640c640..6716528fb8b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -328,8 +328,8 @@ include( ":dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin", ":dd-java-agent:instrumentation:datadog:profiling:enable-wallclock-profiling", ":dd-java-agent:instrumentation:datadog:profiling:exception-profiling", - ":dd-java-agent:instrumentation:datadog:profiling:lock-support-profiling", - ":dd-java-agent:instrumentation:datadog:profiling:object-wait-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:lock-support", + ":dd-java-agent:instrumentation:datadog:profiling:object-wait", ":dd-java-agent:instrumentation:datadog:tracing:trace-annotation", ":dd-java-agent:instrumentation:datanucleus-4.0.5", ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0",