diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 739c09280de..efbaab125d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,6 +64,10 @@ variables: description: "Enable flaky tests" value: "false" + JAVA_PROFILER_REF: + description: "When non-empty, clone DataDog/java-profiler at this Git ref (branch or tag), build ddprof, and use it as ddprof.jar for Gradle jobs instead of the Maven dependency." + value: "paul.fournillon/wallclock_precheck" + # One pipeline injection package size ratchet OCI_PACKAGE_MAX_SIZE_BYTES: 40_000_000 LIB_INJECTION_IMAGE_MAX_SIZE_BYTES: 40_000_000 @@ -167,9 +171,21 @@ default: echo "Failed to find base ref for PR" >&2 fi +# When build_java_profiler_ddprof ran, its artifact is available at custom-ddprof/ddprof.jar. +# Append root project property expected by dd-java-agent/ddprof-lib/build.gradle. +.inject_custom_ddprof_jar: &inject_custom_ddprof_jar + - | + if [ -f "${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" ]; then + echo "ddprof.jar=${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" >> gradle.properties + echo "Using custom ddprof.jar from java-profiler build" + fi + .gradle_build: &gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build + needs: + - job: build_java_profiler_ddprof + optional: true variables: MAVEN_OPTS: "-Xms256M -Xmx1024M" GRADLE_WORKERS: 6 @@ -219,6 +235,7 @@ default: org.gradle.java.installations.auto-download=false org.gradle.java.installations.fromEnv=$JAVA_HOMES EOF + - *inject_custom_ddprof_jar - mkdir -p .gradle - export GRADLE_USER_HOME=$(pwd)/.gradle # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties @@ -294,8 +311,73 @@ dd-octo-sts-pre-release-check: max: 2 when: always +# Builds java-profiler from JAVA_PROFILER_REF and publishes custom-ddprof/ddprof.jar for downstream Gradle jobs. +# Uses :ddprof-lib:assembleReleaseJar (not assembleRelease, which is native-only). JDK 21+ for release + JDK 17+ for Gradle 9. +build_java_profiler_ddprof: + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base + stage: build + rules: + - if: '$JAVA_PROFILER_REF =~ /.+/' + when: on_success + variables: + FF_USE_FASTZIP: "true" + CACHE_COMPRESSION_LEVEL: "slowest" + KUBERNETES_CPU_REQUEST: 10 + KUBERNETES_MEMORY_REQUEST: 20Gi + KUBERNETES_MEMORY_LIMIT: 20Gi + before_script: + - | + # java-profiler uses Gradle 9.x; Gradle requires JVM 17+. Builder image default java is often JDK 8. + if [ -n "${JAVA_21_HOME:-}" ] && [ -x "${JAVA_21_HOME}/bin/java" ]; then + export JAVA_HOME="$JAVA_21_HOME" + elif [ -n "${JAVA_17_HOME:-}" ] && [ -x "${JAVA_17_HOME}/bin/java" ]; then + export JAVA_HOME="$JAVA_17_HOME" + else + shopt -s nullglob + for d in /usr/lib/jvm/java-21-* /usr/lib/jvm/temurin-21-* /usr/lib/jvm/java-17-*; do + if [ -x "${d}/bin/java" ]; then + export JAVA_HOME="$d" + break + fi + done + shopt -u nullglob + fi + if [ -z "${JAVA_HOME:-}" ] || ! [ -x "${JAVA_HOME}/bin/java" ]; then + echo "Could not find JDK 17+ for Gradle 9 (set JAVA_21_HOME or JAVA_17_HOME, or install JDK 21 under /usr/lib/jvm)." >&2 + ls -la /usr/lib/jvm 2>/dev/null || true + exit 1 + fi + export PATH="${JAVA_HOME}/bin:${PATH}" + java -version + script: + - | + set -euo pipefail + mkdir -p "${CI_PROJECT_DIR}/custom-ddprof" + SRCDIR="${CI_PROJECT_DIR}/java-profiler-src" + rm -rf "$SRCDIR" + git clone --depth 1 --branch "$JAVA_PROFILER_REF" https://github.com/DataDog/java-profiler.git "$SRCDIR" + cd "$SRCDIR" + chmod +x ./gradlew + ./gradlew --version + # assembleRelease is the native link/assemble task only; the packaged jar is assembleReleaseJar. + ./gradlew :ddprof-lib:assembleReleaseJar -Pskip-tests -Pskip-gtest + JAR=$(find ddprof-lib/build/libs -maxdepth 1 -type f \( -name 'ddprof-*.jar' \) ! -name '*-sources*' ! -name '*-javadoc*' | head -1) + if [ -z "$JAR" ] || [ ! -f "$JAR" ]; then + echo "No ddprof jar found under ddprof-lib/build/libs" >&2 + ls -la ddprof-lib/build/libs 2>/dev/null || ls -laR ddprof-lib/build 2>/dev/null || true + exit 1 + fi + cp "$JAR" "${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" + ls -la "${CI_PROJECT_DIR}/custom-ddprof/" + artifacts: + when: on_success + paths: + - custom-ddprof/ddprof.jar + build: needs: + - job: build_java_profiler_ddprof + optional: true - job: maven-central-pre-release-check optional: true - job: dd-octo-sts-pre-release-check @@ -406,7 +488,9 @@ publish-artifacts-to-s3: spotless: extends: .gradle_build stage: tests - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true variables: GRADLE_MEMORY_MAX: 6G script: @@ -416,7 +500,9 @@ spotless: check-instrumentation-naming: extends: .gradle_build stage: tests - needs: [ ] + needs: + - job: build_java_profiler_ddprof + optional: true script: - ./gradlew --version - ./gradlew checkInstrumentationNaming @@ -424,7 +510,9 @@ check-instrumentation-naming: config-inversion-linter: extends: .gradle_build stage: tests - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true script: - ./gradlew --version - ./gradlew checkConfigurations @@ -433,7 +521,10 @@ test_published_artifacts: extends: .gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests stage: tests - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" script: @@ -460,7 +551,10 @@ test_published_artifacts: .check_job: extends: .gradle_build - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build stage: tests variables: CACHE_TYPE: "lib" @@ -496,7 +590,9 @@ test_published_artifacts: check_build_src: extends: .check_job - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true variables: GRADLE_TARGET: ":buildSrc:build" @@ -531,7 +627,10 @@ check_debugger: muzzle: extends: .gradle_build - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests parallel: matrix: @@ -563,7 +662,10 @@ muzzle: muzzle-dep-report: extends: .gradle_build - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests variables: CACHE_TYPE: "inst" @@ -600,7 +702,10 @@ muzzle-dep-report: extends: .gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests variables: GRADLE_PARAMS: "-PskipFlakyTests" @@ -898,7 +1003,10 @@ deploy_to_di_backend:manual: deploy_to_maven_central: extends: .gradle_build stage: publish - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" rules: @@ -926,7 +1034,10 @@ deploy_to_maven_central: deploy_snapshot_with_ddprof_snapshot: extends: .gradle_build stage: publish - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" rules: diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index b1e07b08c32..8abf48b769b 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -341,6 +341,17 @@ public void recordTraceRoot(long rootSpanId, String endpoint, String operation) } } + /** Monotonic tick count for TaskBlock and wall-clock off-CPU interval timing. */ + public long getCurrentTicks() { + return profiler.getCurrentTicks(); + } + + int encode(CharSequence constant) { + // java-profiler ContextSetter no longer exposes value encoding. + // Keep API contract by returning "not encoded" (0), which callers already handle. + return 0; + } + public int operationNameOffset() { return offsetOf(OPERATION); } @@ -454,4 +465,24 @@ void recordQueueTimeEvent( } } } + + void recordTaskBlockEvent( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + if (profiler != null) { + long endTicks = profiler.getCurrentTicks(); + profiler.recordTaskBlock(startTicks, endTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + } + + void parkEnter(long spanId, long rootSpanId) { + if (profiler != null) { + profiler.parkEnter(spanId, rootSpanId); + } + } + + void parkExit(long blocker, long unblockingSpanId) { + if (profiler != null) { + profiler.parkExit(blocker, unblockingSpanId); + } + } } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 00a0358d346..67af75a7afd 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -71,11 +71,53 @@ public void onDetach() { } } + @Override + public int encode(CharSequence constant) { + return DDPROF.encode(constant); + } + + @Override + public int encodeOperationName(CharSequence constant) { + if (SPAN_NAME_INDEX >= 0) { + return DDPROF.encode(constant); + } + return 0; + } + + @Override + public int encodeResourceName(CharSequence constant) { + if (RESOURCE_NAME_INDEX >= 0) { + return DDPROF.encode(constant); + } + return 0; + } + @Override public String name() { return "ddprof"; } + @Override + public long getCurrentTicks() { + return DDPROF.getCurrentTicks(); + } + + @Override + public void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + DDPROF.recordTaskBlockEvent(startTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + + @Override + public void parkEnter(long spanId, long rootSpanId) { + DDPROF.parkEnter(spanId, rootSpanId); + } + + @Override + public void parkExit(long blocker, long unblockingSpanId) { + DDPROF.parkExit(blocker, unblockingSpanId); + } + public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index 55d39ba52a0..80754c7937f 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.Properties; import java.util.UUID; +import java.util.concurrent.locks.LockSupport; import java.util.stream.IntStream; import java.util.stream.Stream; import org.junit.jupiter.api.Assumptions; @@ -29,6 +30,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.ItemFilters; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,6 +77,47 @@ void test() throws Exception { } } + @Test + void testTaskBlockBridgeMethodsEmitTaskBlockEvents() throws Exception { + assertDoesNotThrow( + () -> DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(), "Profiler not available"); + DatadogProfiler profiler = DatadogProfiler.newInstance(ConfigProvider.getInstance()); + if (profiler.isActive()) { + log.warn("Datadog profiler is already running. Skipping task-block integration test."); + return; + } + + OngoingRecording recording = profiler.start(); + if (recording == null) { + log.warn("Datadog Profiler is not available. Skipping task-block integration test."); + return; + } + + try { + // Direct bridge path (recordTaskBlock -> JavaProfiler.recordTaskBlock0) + long startTicks = profiler.getCurrentTicks(); + LockSupport.parkNanos(3_000_000L); // > 1ms native threshold + profiler.recordTaskBlockEvent(startTicks, 101L, 202L, 303L, 404L); + + // Park path (parkEnter/parkExit -> JavaProfiler.parkEnter0/parkExit0) + profiler.parkEnter(505L, 606L); + LockSupport.parkNanos(3_000_000L); // > 1ms native threshold + profiler.parkExit(707L, 808L); + + RecordingData data = profiler.stop(recording); + assertNotNull(data); + IItemCollection events = JfrLoaderToolkit.loadEvents(data.getStream()); + long taskBlockCount = + events.apply(ItemFilters.type("datadog.TaskBlock")).stream() + .mapToLong(IItemIterable::getItemCount) + .sum(); + + assertTrue(taskBlockCount > 0, "Expected datadog.TaskBlock events from bridge methods"); + } finally { + recording.stop(); + } + } + @ParameterizedTest @MethodSource("profilingModes") void testStartCmd(boolean cpu, boolean wall, boolean alloc, boolean memleak) throws Exception { diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle new file mode 100644 index 00000000000..9935e818b49 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle @@ -0,0 +1,11 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + coreJdk() + } +} + +dependencies { + testImplementation libs.bundles.junit5 +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java new file mode 100644 index 00000000000..c6607ed5f16 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -0,0 +1,131 @@ +package datadog.trace.instrumentation.locksupport; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isStatic; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import java.util.concurrent.ConcurrentHashMap; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.matcher.ElementMatchers; + +/** + * Instruments {@link java.util.concurrent.locks.LockSupport#park} variants to emit {@code + * datadog.TaskBlock} JFR events. These events record the span, root-span, and duration of every + * blocking interval, enabling critical-path analysis across async handoffs. + * + *

Also instruments {@link java.util.concurrent.locks.LockSupport#unpark} to capture the span ID + * of the unblocking thread, which is then recorded in the TaskBlock event. + * + *

{@code parkEnter} runs even without an active span (span id 0) so the native wall-clock + * precheck can suppress {@code SIGVTALRM} for the whole park interval. TaskBlock JFR emission is + * gated by the profiler on duration and span context. + */ +@AutoService(InstrumenterModule.class) +public class LockSupportProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + public LockSupportProfilingInstrumentation() { + super("lock-support"); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.util.concurrent.locks.LockSupport"}; + } + + @Override + public String[] muzzleIgnoredClassNames() { + // Advice references this nested holder; it lives on the instrumentation classpath only. + return new String[] {getClass().getName() + "$State"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(nameStartsWith("park")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$ParkAdvice"); + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(ElementMatchers.named("unpark")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$UnparkAdvice"); + } + + /** Holds shared state accessible from both {@link ParkAdvice} and {@link UnparkAdvice}. */ + public static final class State { + /** Maps target thread to the span ID of the thread that called {@code unpark()} on it. */ + public static final ConcurrentHashMap UNPARKING_SPAN = new ConcurrentHashMap<>(); + } + + public static final class ParkAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static long[] before(@Advice.Argument(value = 0, optional = true) Object blocker) { + ProfilingContextIntegration profiling = AgentTracer.get().getProfilingContext(); + if (profiling == null) { + return null; + } + // Always call parkEnter for signal suppression, even without an active span. + // spanId/rootSpanId = 0 when no active span — no JFR event will be emitted at exit for + // zero-span intervals (the native code filters those out by duration threshold and span + // check). + long spanId = 0L; + long rootSpanId = 0L; + AgentSpan span = AgentTracer.activeSpan(); + if (span != null && span.context() instanceof ProfilerContext) { + ProfilerContext ctx = (ProfilerContext) span.context(); + spanId = ctx.getSpanId(); + rootSpanId = ctx.getRootSpanId(); + } + profiling.parkEnter(spanId, rootSpanId); + long blockerHash = blocker != null ? System.identityHashCode(blocker) : 0L; + return new long[] {blockerHash, spanId, rootSpanId}; + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter long[] state) { + // Always drain the map entry before any early return. If we returned first, a stale + // unblocking-span ID placed by a prior unpark() would persist and be incorrectly + // attributed to the next TaskBlock event emitted on this thread. + Long unblockingSpanId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + if (state == null) { + return; + } + // state = [blockerHash, spanId, rootSpanId] — set in before(). + // parkExit() records the TaskBlock JFR event using the start tick saved by parkEnter(). + AgentTracer.get() + .getProfilingContext() + .parkExit(state[0], unblockingSpanId != null ? unblockingSpanId : 0L); + } + } + + public static final class UnparkAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before(@Advice.Argument(0) Thread thread) { + if (thread == null) { + return; + } + AgentSpan span = AgentTracer.activeSpan(); + if (span == null || !(span.context() instanceof ProfilerContext)) { + return; + } + ProfilerContext ctx = (ProfilerContext) span.context(); + long effectiveSpanId = ctx.getSpanId(); + State.UNPARKING_SPAN.put(thread, effectiveSpanId); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java new file mode 100644 index 00000000000..c2a16b13868 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java @@ -0,0 +1,190 @@ +package datadog.trace.instrumentation.locksupport; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.instrumentation.locksupport.LockSupportProfilingInstrumentation.State; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LockSupportProfilingInstrumentation}. + * + *

These tests exercise the {@link State} map directly, verifying the mechanism used to + * communicate the unblocking span ID from {@code UnparkAdvice} to {@code ParkAdvice}. + */ +class LockSupportProfilingInstrumentationTest { + + @BeforeEach + void clearState() { + State.UNPARKING_SPAN.clear(); + } + + @AfterEach + void cleanupState() { + State.UNPARKING_SPAN.clear(); + } + + // ------------------------------------------------------------------------- + // State map — basic contract + // ------------------------------------------------------------------------- + + @Test + void state_put_and_remove() { + Thread t = Thread.currentThread(); + long spanId = 12345L; + + State.UNPARKING_SPAN.put(t, spanId); + Long retrieved = State.UNPARKING_SPAN.remove(t); + + assertNotNull(retrieved); + assertEquals(spanId, (long) retrieved); + // After removal the entry should be gone + assertNull(State.UNPARKING_SPAN.get(t)); + } + + @Test + void state_remove_returns_null_when_absent() { + Thread t = new Thread(() -> {}); + assertNull(State.UNPARKING_SPAN.remove(t)); + } + + @Test + void state_is_initially_empty() { + assertTrue(State.UNPARKING_SPAN.isEmpty()); + } + + // ------------------------------------------------------------------------- + // Multithreaded: unpark thread populates map, parked thread reads it + // ------------------------------------------------------------------------- + + /** + * Simulates the UnparkAdvice → ParkAdvice handoff: + * + *

    + *
  1. Thread A (the "parked" thread) blocks on a latch. + *
  2. Thread B (the "unparking" thread) places its span ID in {@code State.UNPARKING_SPAN} for + * Thread A and then releases the latch. + *
  3. Thread A wakes up, reads and removes the span ID from the map. + *
+ */ + @Test + void unparking_spanId_is_visible_to_parked_thread() throws InterruptedException { + long unparkingSpanId = 99887766L; + + CountDownLatch ready = new CountDownLatch(1); + CountDownLatch go = new CountDownLatch(1); + AtomicLong capturedSpanId = new AtomicLong(-1L); + AtomicReference parkedThreadRef = new AtomicReference<>(); + + Thread parkedThread = + new Thread( + () -> { + parkedThreadRef.set(Thread.currentThread()); + ready.countDown(); + try { + go.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Simulate what ParkAdvice.after does: read and remove unblocking span id + Long unblockingId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + + parkedThread.start(); + ready.await(); // wait for parked thread to register itself + + // Simulate what UnparkAdvice.before does: record unparking span id + State.UNPARKING_SPAN.put(parkedThread, unparkingSpanId); + go.countDown(); // unblock parked thread + + parkedThread.join(2_000); + assertFalse(parkedThread.isAlive(), "Test thread did not finish in time"); + assertEquals( + unparkingSpanId, + capturedSpanId.get(), + "Parked thread should have read the unblocking span id placed by unparking thread"); + } + + /** + * Verifies that if no entry exists for the parked thread (i.e. the thread was unblocked by a + * non-traced thread), the {@code remove} returns {@code null} and the code falls back to 0. + */ + @Test + void no_unparking_entry_yields_zero() throws InterruptedException { + AtomicLong capturedSpanId = new AtomicLong(-1L); + + Thread parkedThread = + new Thread( + () -> { + Long unblockingId = State.UNPARKING_SPAN.remove(Thread.currentThread()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + parkedThread.start(); + parkedThread.join(2_000); + + assertEquals( + 0L, capturedSpanId.get(), "Should fall back to 0 when no unparking span id is recorded"); + } + + // ------------------------------------------------------------------------- + // ParkAdvice.after — null state is a no-op + // ------------------------------------------------------------------------- + + /** + * When {@code ParkAdvice.before} returns {@code null} (profiler not active or no active span), + * {@code ParkAdvice.after} must not throw and must not leave entries in {@code UNPARKING_SPAN}. + * It does call {@code remove(currentThread)}, but on an empty map that is a no-op. + */ + @Test + void parkAdvice_after_null_state_isNoOp() { + LockSupportProfilingInstrumentation.ParkAdvice.after(null); + assertTrue(State.UNPARKING_SPAN.isEmpty()); + } + + /** + * Regression test for stale-entry misattribution. + * + *

If {@code unpark(t)} is called (inserting an entry into {@code UNPARKING_SPAN}) and thread + * {@code t} then parks without an active span ({@code state == null}), the entry must still be + * drained. Without the fix, it would linger and be incorrectly attributed to the next {@code + * TaskBlock} emitted on that thread. + */ + @Test + void stale_entry_is_drained_when_park_fires_without_active_span() { + Thread t = Thread.currentThread(); + State.UNPARKING_SPAN.put(t, 99L); + + // Simulate park() returning with no active span (state == null) + LockSupportProfilingInstrumentation.ParkAdvice.after(null); + + assertNull( + State.UNPARKING_SPAN.get(t), + "Stale UNPARKING_SPAN entry must be drained even when state is null"); + } + + /** + * If multiple unpark calls race for the same parked thread, the latest span ID should be consumed + * and the entry must still be drained exactly once by ParkAdvice.after(). + */ + @Test + void latest_unparking_span_wins_and_entry_is_drained() { + Thread t = Thread.currentThread(); + State.UNPARKING_SPAN.put(t, 101L); + State.UNPARKING_SPAN.put(t, 202L); + + Long consumed = State.UNPARKING_SPAN.remove(t); + assertNotNull(consumed); + assertEquals(202L, consumed.longValue()); + assertNull(State.UNPARKING_SPAN.get(t), "Entry must be removed after consumption"); + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle new file mode 100644 index 00000000000..d61e8bd6155 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle @@ -0,0 +1,16 @@ +apply from: "$rootDir/gradle/java.gradle" + +testJvmConstraints { + minJavaVersion = JavaVersion.VERSION_21 +} + +muzzle { + pass { + coreJdk('21') + } +} + +dependencies { + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java new file mode 100644 index 00000000000..a96357fe4a9 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java @@ -0,0 +1,142 @@ +package datadog.trace.instrumentation.objectwait; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import datadog.environment.JavaVirtualMachine; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import net.bytebuddy.asm.Advice; + +/** + * Instruments {@link Object#wait(long)} in JDK 21+ to emit {@code datadog.TaskBlock} JFR events. + * + *

In JDK 21+, {@code wait(long)} is a pure-Java wrapper around the native {@code wait0(long)}, + * so ByteBuddy can add advice to it. In JDK 8-20 the method is declared {@code native} and is not + * instrumented by this class (Approach 1 osThreadState precheck already suppresses SIGVTALRM for + * threads in OBJECT_WAIT state on all JDK versions). + * + *

Only {@code wait(long)} is instrumented: {@code wait()} delegates to {@code wait(0L)} and + * {@code wait(long, int)} delegates to {@code wait(long)}, so all wait variants are covered. + * + *

{@code unblockingSpanId} is always 0 because {@code notify()} and {@code notifyAll()} remain + * {@code native} in JDK 21+ and the notifying thread cannot be identified via BCI. + */ +@AutoService(InstrumenterModule.class) +public class ObjectWaitProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + public ObjectWaitProfilingInstrumentation() { + super("object-wait"); + } + + @Override + public boolean isEnabled() { + return JavaVirtualMachine.isJavaVersionAtLeast(21) && super.isEnabled(); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.lang.Object"}; + } + + @Override + public String[] muzzleIgnoredClassNames() { + // Static helpers on the advice class produce intra-class references that core-JDK muzzle + // cannot resolve against an empty application classpath. + return new String[] {getClass().getName() + "$WaitAdvice"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(named("wait")) + .and(takesArguments(1)) + .and(takesArgument(0, long.class)) + .and(isDeclaredBy(named("java.lang.Object"))), + getClass().getName() + "$WaitAdvice"); + } + + public static final class WaitAdvice { + + // 1 ms — matches the default LockSupport park threshold in parkExit0 + static final long MIN_WAIT_NANOS = 1_000_000L; + + // State array indices — package-private for readability in tests + static final int IDX_BLOCKER = 0; + static final int IDX_SPAN_ID = 1; + static final int IDX_ROOT_SPAN_ID = 2; + static final int IDX_START_TICKS = 3; + static final int IDX_START_NANOS = 4; + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static long[] before(@Advice.This Object monitor) { + return captureState( + monitor, AgentTracer.get().getProfilingContext(), AgentTracer.activeSpan()); + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter long[] state) { + if (state == null) { + return; + } + ProfilingContextIntegration profiling = AgentTracer.get().getProfilingContext(); + if (profiling == null) { + return; + } + emitIfLongEnough(state, profiling); + } + + /** + * Captures wait-entry state. Package-private to allow direct unit-testing without a live agent. + * + * @return a 5-element {@code long[]} on success, or {@code null} when the preconditions are not + * met (no profiling context, no active span, or span context is not a {@link + * ProfilerContext}) + */ + static long[] captureState( + Object monitor, ProfilingContextIntegration profiling, AgentSpan span) { + if (profiling == null) { + return null; + } + if (span == null || !(span.context() instanceof ProfilerContext)) { + return null; + } + ProfilerContext ctx = (ProfilerContext) span.context(); + return new long[] { + System.identityHashCode(monitor), // [IDX_BLOCKER] monitor identity + ctx.getSpanId(), // [IDX_SPAN_ID] + ctx.getRootSpanId(), // [IDX_ROOT_SPAN_ID] + profiling.getCurrentTicks(), // [IDX_START_TICKS] TSC ticks for event timing + System.nanoTime() // [IDX_START_NANOS] wall-clock nanos for duration filter + }; + } + + /** + * Emits a TaskBlock event if the elapsed wall time since entry exceeds {@link #MIN_WAIT_NANOS}. + * Package-private to allow direct unit-testing without a live agent. + */ + static void emitIfLongEnough(long[] state, ProfilingContextIntegration profiling) { + if (System.nanoTime() - state[IDX_START_NANOS] < MIN_WAIT_NANOS) { + return; + } + // unblockingSpanId = 0: notify/notifyAll are native in JDK 21+, + // so the notifying thread cannot be identified via BCI. + profiling.recordTaskBlock( + state[IDX_START_TICKS], + state[IDX_SPAN_ID], + state[IDX_ROOT_SPAN_ID], + state[IDX_BLOCKER], + 0L); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java new file mode 100644 index 00000000000..e1e4f158fa2 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java @@ -0,0 +1,204 @@ +package datadog.trace.instrumentation.objectwait; + +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_BLOCKER; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_ROOT_SPAN_ID; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_SPAN_ID; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_START_NANOS; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.IDX_START_TICKS; +import static datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice.MIN_WAIT_NANOS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +// Combining both interfaces as span.context() return type is AgentSpanContext, +// but production code casts it to ProfilerContext — both must be satisfied by the mock. +// AgentSpanContext is the declared return type; ProfilerContext provides getSpanId/getRootSpanId. + +/** + * Unit tests for {@link ObjectWaitProfilingInstrumentation}. + * + *

These tests exercise the two package-private helpers ({@code captureState} and {@code + * emitIfLongEnough}) directly, bypassing the {@code AgentTracer.get()} call that would require a + * live agent. The {@link WaitAdvice#after(long[])} null-guard is tested separately. + */ +@ExtendWith(MockitoExtension.class) +class ObjectWaitProfilingInstrumentationTest { + + private static final long SPAN_ID = 0xDEADBEEFL; + private static final long ROOT_SPAN_ID = 0xCAFEBABEL; + private static final long START_TICKS = 42_000_000L; + + /** + * Implements both interfaces: span.context() returns AgentSpanContext but is cast to + * ProfilerContext. + */ + private interface ProfilerSpanContext extends AgentSpanContext, ProfilerContext {} + + @Mock private ProfilingContextIntegration profiling; + @Mock private AgentSpan span; + @Mock private ProfilerSpanContext ctx; + @Mock private AgentSpanContext nonProfilerCtx; + + private final Object monitor = new Object(); + + @BeforeEach + void setUp() { + lenient().when(span.context()).thenReturn(ctx); + lenient().when(ctx.getSpanId()).thenReturn(SPAN_ID); + lenient().when(ctx.getRootSpanId()).thenReturn(ROOT_SPAN_ID); + lenient().when(profiling.getCurrentTicks()).thenReturn(START_TICKS); + } + + // --------------------------------------------------------------------------- + // WaitAdvice.after — null state guard + // --------------------------------------------------------------------------- + + @Test + void after_nullState_doesNotThrow() { + // before() returns null when profiling is absent or no active span; + // after() must be a safe no-op in that case. + WaitAdvice.after(null); + } + + // --------------------------------------------------------------------------- + // captureState — precondition gates + // --------------------------------------------------------------------------- + + @Test + void captureState_nullProfiling_returnsNull() { + assertNull(WaitAdvice.captureState(monitor, null, span)); + } + + @Test + void captureState_nullSpan_returnsNull() { + assertNull(WaitAdvice.captureState(monitor, profiling, null)); + } + + @Test + void captureState_spanContextNotProfilerContext_returnsNull() { + lenient().when(span.context()).thenReturn(nonProfilerCtx); + assertNull(WaitAdvice.captureState(monitor, profiling, span)); + } + + // --------------------------------------------------------------------------- + // captureState — happy path + // --------------------------------------------------------------------------- + + @Test + void captureState_validSpan_returnsCorrectState() { + long before = System.nanoTime(); + long[] state = WaitAdvice.captureState(monitor, profiling, span); + long after = System.nanoTime(); + + assertNotNull(state); + assertEquals(5, state.length); + assertEquals(System.identityHashCode(monitor), state[IDX_BLOCKER]); + assertEquals(SPAN_ID, state[IDX_SPAN_ID]); + assertEquals(ROOT_SPAN_ID, state[IDX_ROOT_SPAN_ID]); + assertEquals(START_TICKS, state[IDX_START_TICKS]); + // startNanos must be within the window captured around the call + org.junit.jupiter.api.Assertions.assertTrue( + state[IDX_START_NANOS] >= before && state[IDX_START_NANOS] <= after, + "startNanos should be captured during captureState()"); + } + + @Test + void captureState_differentMonitors_produceDifferentBlockerHashes() { + Object monitorA = new Object(); + Object monitorB = new Object(); + + long[] stateA = WaitAdvice.captureState(monitorA, profiling, span); + long[] stateB = WaitAdvice.captureState(monitorB, profiling, span); + + assertNotNull(stateA); + assertNotNull(stateB); + // Different object identities must produce different blocker hashes + // (identity hash codes can collide, but for freshly allocated objects this holds + // in all current JVM implementations — and what matters is they use identityHashCode) + assertEquals(System.identityHashCode(monitorA), stateA[IDX_BLOCKER]); + assertEquals(System.identityHashCode(monitorB), stateB[IDX_BLOCKER]); + } + + // --------------------------------------------------------------------------- + // emitIfLongEnough — duration filter + // --------------------------------------------------------------------------- + + @Test + void emitIfLongEnough_shortWait_doesNotEmit() { + // Set startNanos to "just now" so elapsed < MIN_WAIT_NANOS + long[] state = buildState(System.nanoTime()); + + WaitAdvice.emitIfLongEnough(state, profiling); + + verifyNoInteractions(profiling); + } + + @Test + void emitIfLongEnough_longWait_emitsTaskBlock() { + // Set startNanos to 2 ms ago — comfortably above the 1 ms threshold + long twoMsAgo = System.nanoTime() - 2 * MIN_WAIT_NANOS; + long[] state = buildState(twoMsAgo); + + WaitAdvice.emitIfLongEnough(state, profiling); + + verify(profiling) + .recordTaskBlock( + START_TICKS, // startTicks + SPAN_ID, // spanId + ROOT_SPAN_ID, // rootSpanId + System.identityHashCode(monitor), // blocker + 0L); // unblockingSpanId always 0 (notify is native) + } + + @Test + void emitIfLongEnough_exactlyAtThreshold_emitsTaskBlock() { + // nanos exactly at the boundary — elapsed == MIN_WAIT_NANOS is NOT < threshold, so it emits + long atThreshold = System.nanoTime() - MIN_WAIT_NANOS; + long[] state = buildState(atThreshold); + + WaitAdvice.emitIfLongEnough(state, profiling); + + verify(profiling) + .recordTaskBlock(START_TICKS, SPAN_ID, ROOT_SPAN_ID, System.identityHashCode(monitor), 0L); + } + + @Test + void emitIfLongEnough_unblockingSpanIdIsAlwaysZero() { + long twoMsAgo = System.nanoTime() - 2 * MIN_WAIT_NANOS; + long[] state = buildState(twoMsAgo); + + WaitAdvice.emitIfLongEnough(state, profiling); + + // The 5th argument (unblockingSpanId) must always be 0 because notify/notifyAll + // are native in JDK 21+ — the notifying thread cannot be identified via BCI. + verify(profiling).recordTaskBlock(START_TICKS, SPAN_ID, ROOT_SPAN_ID, state[IDX_BLOCKER], 0L); + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + private long[] buildState(long startNanos) { + return new long[] { + System.identityHashCode(monitor), // IDX_BLOCKER + SPAN_ID, // IDX_SPAN_ID + ROOT_SPAN_ID, // IDX_ROOT_SPAN_ID + START_TICKS, // IDX_START_TICKS + startNanos // IDX_START_NANOS + }; + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java index 4accced983a..3e67cf48fa3 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java @@ -34,6 +34,36 @@ default int encodeResourceName(CharSequence constant) { return 0; } + /** Returns the current TSC tick count for the calling thread. */ + default long getCurrentTicks() { + return 0L; + } + + /** + * Emits a TaskBlock event covering a blocking interval on the current thread. + * + * @param startTicks TSC tick at block entry + * @param spanId the span ID active when blocking began + * @param rootSpanId the local root span ID active when blocking began + * @param blocker identity hash code of the blocking object, or 0 if none + * @param unblockingSpanId the span ID of the thread that unblocked this thread, or 0 if unknown + */ + default void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) {} + + /** + * Called when the current thread is about to enter {@code LockSupport.park*}. Native code can + * suppress wall-clock signals for the park interval and record the start tick for off-CPU + * analysis. + */ + default void parkEnter(long spanId, long rootSpanId) {} + + /** + * Called when the current thread has returned from {@code LockSupport.park*}. Clears the park + * state and may emit a TaskBlock JFR event. + */ + default void parkExit(long blocker, long unblockingSpanId) {} + String name(); final class NoOp implements ProfilingContextIntegration { diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index d77b600b2f6..7ac7866be94 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7865,6 +7865,14 @@ "aliases": ["DD_TRACE_INTEGRATION_LIBERTY_ENABLED", "DD_INTEGRATION_LIBERTY_ENABLED"] } ], + "DD_TRACE_LOCK_SUPPORT_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_LOCK_SUPPORT_ENABLED", "DD_INTEGRATION_LOCK_SUPPORT_ENABLED"] + } + ], "DD_TRACE_LOG4J_1_ENABLED": [ { "version": "A", @@ -8289,6 +8297,14 @@ "aliases": ["DD_OBFUSCATION_QUERY_STRING_REGEXP"] } ], + "DD_TRACE_OBJECT_WAIT_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_OBJECT_WAIT_ENABLED", "DD_INTEGRATION_OBJECT_WAIT_ENABLED"] + } + ], "DD_TRACE_OGNL_ENABLED": [ { "version": "A", diff --git a/settings.gradle.kts b/settings.gradle.kts index bd5aaceffaa..6716528fb8b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -328,6 +328,8 @@ include( ":dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin", ":dd-java-agent:instrumentation:datadog:profiling:enable-wallclock-profiling", ":dd-java-agent:instrumentation:datadog:profiling:exception-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:lock-support", + ":dd-java-agent:instrumentation:datadog:profiling:object-wait", ":dd-java-agent:instrumentation:datadog:tracing:trace-annotation", ":dd-java-agent:instrumentation:datanucleus-4.0.5", ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0",