diff --git a/.github/workflows/run-system-tests.yaml b/.github/workflows/run-system-tests.yaml index 2c441f495de..23dba04fc53 100644 --- a/.github/workflows/run-system-tests.yaml +++ b/.github/workflows/run-system-tests.yaml @@ -22,6 +22,11 @@ jobs: build: runs-on: group: APM Larger Runners + # Keep in sync with the JAVA_PROFILER_REF default in .gitlab-ci.yml. When non-empty, + # we clone DataDog/java-profiler at this ref, build :ddprof-lib:assembleReleaseJar + # and inject it via -Pddprof.jar= instead of the published Maven artifact. + env: + JAVA_PROFILER_REF: paul.fournillon/wallclock_precheck steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 @@ -39,17 +44,51 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- + # Mirrors the `build_java_profiler_ddprof` GitLab job: clone java-profiler at + # JAVA_PROFILER_REF, build :ddprof-lib:assembleReleaseJar with JDK 21 (Gradle 9.x), + # and stage the resulting jar under custom-ddprof/ddprof.jar. + # assembleRelease is the native link/assemble task only; the packaged jar is assembleReleaseJar. + - name: Build custom ddprof.jar from java-profiler + if: ${{ env.JAVA_PROFILER_REF != '' }} + run: | + set -euo pipefail + mkdir -p custom-ddprof + SRCDIR="${RUNNER_TEMP}/java-profiler-src" + rm -rf "$SRCDIR" + git clone --depth 1 --branch "$JAVA_PROFILER_REF" https://github.com/DataDog/java-profiler.git "$SRCDIR" + ( + cd "$SRCDIR" + chmod +x ./gradlew + JAVA_HOME="$JAVA_HOME_21_X64" PATH="$JAVA_HOME_21_X64/bin:$PATH" ./gradlew --version + JAVA_HOME="$JAVA_HOME_21_X64" PATH="$JAVA_HOME_21_X64/bin:$PATH" \ + ./gradlew :ddprof-lib:assembleReleaseJar -Pskip-tests -Pskip-gtest --no-daemon + ) + JAR=$(find "$SRCDIR/ddprof-lib/build/libs" -maxdepth 1 -type f -name 'ddprof-*.jar' ! -name '*-sources*' ! -name '*-javadoc*' | head -1) + if [ -z "$JAR" ] || [ ! -f "$JAR" ]; then + echo "No ddprof jar found under $SRCDIR/ddprof-lib/build/libs" >&2 + ls -laR "$SRCDIR/ddprof-lib/build" 2>/dev/null || true + exit 1 + fi + cp "$JAR" "$GITHUB_WORKSPACE/custom-ddprof/ddprof.jar" + echo "DDPROF_JAR=$GITHUB_WORKSPACE/custom-ddprof/ddprof.jar" >> "$GITHUB_ENV" + ls -la "$GITHUB_WORKSPACE/custom-ddprof/" + - name: Build dd-trace-java env: ORG_GRADLE_PROJECT_akkaRepositoryToken: ${{ secrets.AKKA_REPO_TOKEN }} run: | + DDPROF_ARG="" + if [ -n "${DDPROF_JAR:-}" ] && [ -f "$DDPROF_JAR" ]; then + echo "Injecting custom ddprof.jar: $DDPROF_JAR" + DDPROF_ARG="-Pddprof.jar=$DDPROF_JAR" + fi GRADLE_OPTS="-Xms2g -Xmx4g -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC" \ JAVA_HOME=$JAVA_HOME_8_X64 \ JAVA_8_HOME=$JAVA_HOME_8_X64 \ JAVA_11_HOME=$JAVA_HOME_11_X64 \ JAVA_17_HOME=$JAVA_HOME_17_X64 \ JAVA_21_HOME=$JAVA_HOME_21_X64 \ - ./gradlew clean :dd-java-agent:shadowJar \ + ./gradlew clean :dd-java-agent:shadowJar $DDPROF_ARG \ --build-cache --parallel --stacktrace --no-daemon --max-workers=4 - name: Upload artifact diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5290d93abd0..d72416cbb99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,6 +66,10 @@ variables: description: "Enable flaky tests" value: "false" + JAVA_PROFILER_REF: + description: "When non-empty, clone DataDog/java-profiler at this Git ref (branch or tag), build ddprof, and use it as ddprof.jar for Gradle jobs instead of the Maven dependency." + value: "paul.fournillon/wallclock_precheck" + # One pipeline injection package size ratchet OCI_PACKAGE_MAX_SIZE_BYTES: 40_000_000 LIB_INJECTION_IMAGE_MAX_SIZE_BYTES: 40_000_000 @@ -172,9 +176,21 @@ default: echo "Failed to find base ref for PR" >&2 fi +# When build_java_profiler_ddprof ran, its artifact is available at custom-ddprof/ddprof.jar. +# Append root project property expected by dd-java-agent/ddprof-lib/build.gradle. +.inject_custom_ddprof_jar: &inject_custom_ddprof_jar + - | + if [ -f "${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" ]; then + echo "ddprof.jar=${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" >> gradle.properties + echo "Using custom ddprof.jar from java-profiler build" + fi + .gradle_build: &gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build + needs: + - job: build_java_profiler_ddprof + optional: true variables: MAVEN_OPTS: "-Xms256M -Xmx1024M" GRADLE_WORKERS: 6 @@ -224,6 +240,7 @@ default: org.gradle.java.installations.auto-download=false org.gradle.java.installations.fromEnv=$JAVA_HOMES EOF + - *inject_custom_ddprof_jar - mkdir -p .gradle - export GRADLE_USER_HOME=$(pwd)/.gradle # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties @@ -293,8 +310,114 @@ dd-octo-sts-pre-release-check: max: 2 when: always +# Builds java-profiler from JAVA_PROFILER_REF and publishes custom-ddprof/ddprof.jar for downstream Gradle jobs. +# Uses :ddprof-lib:assembleReleaseJar (not assembleRelease, which is native-only). JDK 21+ for release + JDK 17+ for Gradle 9. +build_java_profiler_ddprof: + image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base + stage: build + rules: + - if: '$JAVA_PROFILER_REF =~ /.+/' + when: on_success + variables: + FF_USE_FASTZIP: "true" + CACHE_COMPRESSION_LEVEL: "slowest" + KUBERNETES_CPU_REQUEST: 10 + KUBERNETES_MEMORY_REQUEST: 20Gi + KUBERNETES_MEMORY_LIMIT: 20Gi + before_script: + - | + # java-profiler uses Gradle 9.x; Gradle requires JVM 17+. Builder image default java is often JDK 8. + if [ -n "${JAVA_21_HOME:-}" ] && [ -x "${JAVA_21_HOME}/bin/java" ]; then + export JAVA_HOME="$JAVA_21_HOME" + elif [ -n "${JAVA_17_HOME:-}" ] && [ -x "${JAVA_17_HOME}/bin/java" ]; then + export JAVA_HOME="$JAVA_17_HOME" + else + shopt -s nullglob + for d in /usr/lib/jvm/java-21-* /usr/lib/jvm/temurin-21-* /usr/lib/jvm/java-17-*; do + if [ -x "${d}/bin/java" ]; then + export JAVA_HOME="$d" + break + fi + done + shopt -u nullglob + fi + if [ -z "${JAVA_HOME:-}" ] || ! [ -x "${JAVA_HOME}/bin/java" ]; then + echo "Could not find JDK 17+ for Gradle 9 (set JAVA_21_HOME or JAVA_17_HOME, or install JDK 21 under /usr/lib/jvm)." >&2 + ls -la /usr/lib/jvm 2>/dev/null || true + exit 1 + fi + export PATH="${JAVA_HOME}/bin:${PATH}" + java -version + script: + - | + set -euo pipefail + mkdir -p "${CI_PROJECT_DIR}/custom-ddprof" + SRCDIR="${CI_PROJECT_DIR}/java-profiler-src" + rm -rf "$SRCDIR" + git clone --depth 1 --branch "$JAVA_PROFILER_REF" https://github.com/DataDog/java-profiler.git "$SRCDIR" + cd "$SRCDIR" + export ORG_GRADLE_PROJECT_mavenRepositoryProxy="$MAVEN_REPOSITORY_PROXY" + export ORG_GRADLE_PROJECT_gradlePluginProxy="$GRADLE_PLUGIN_PROXY" + PROFILER_GRADLE_INIT="${CI_PROJECT_DIR}/java-profiler-init.gradle" + cat > "$PROFILER_GRADLE_INIT" <<'EOF' + def mavenRepositoryProxy = System.getenv('MAVEN_REPOSITORY_PROXY') + def gradlePluginProxy = System.getenv('GRADLE_PLUGIN_PROXY') ?: mavenRepositoryProxy + + def addPluginRepositories = { repositories -> + if (gradlePluginProxy) { + repositories.maven { url = uri(gradlePluginProxy) } + } + if (mavenRepositoryProxy && mavenRepositoryProxy != gradlePluginProxy) { + repositories.maven { url = uri(mavenRepositoryProxy) } + } + } + + def addMavenRepositories = { repositories -> + if (mavenRepositoryProxy) { + repositories.maven { url = uri(mavenRepositoryProxy) } + } + } + + beforeSettings { settings -> + settings.pluginManagement { + repositories { + addPluginRepositories(delegate) + } + } + } + + allprojects { + buildscript { + repositories { + addPluginRepositories(delegate) + } + } + repositories { + addMavenRepositories(delegate) + } + } + EOF + chmod +x ./gradlew + ./gradlew --version + # assembleRelease is the native link/assemble task only; the packaged jar is assembleReleaseJar. + ./gradlew --init-script "$PROFILER_GRADLE_INIT" :ddprof-lib:assembleReleaseJar -Pskip-tests -Pskip-gtest + JAR=$(find ddprof-lib/build/libs -maxdepth 1 -type f \( -name 'ddprof-*.jar' \) ! -name '*-sources*' ! -name '*-javadoc*' | head -1) + if [ -z "$JAR" ] || [ ! -f "$JAR" ]; then + echo "No ddprof jar found under ddprof-lib/build/libs" >&2 + ls -la ddprof-lib/build/libs 2>/dev/null || ls -laR ddprof-lib/build 2>/dev/null || true + exit 1 + fi + cp "$JAR" "${CI_PROJECT_DIR}/custom-ddprof/ddprof.jar" + ls -la "${CI_PROJECT_DIR}/custom-ddprof/" + artifacts: + when: on_success + paths: + - custom-ddprof/ddprof.jar + build: needs: + - job: build_java_profiler_ddprof + optional: true - job: maven-central-pre-release-check optional: true - job: dd-octo-sts-pre-release-check @@ -405,7 +528,9 @@ publish-artifacts-to-s3: spotless: extends: .gradle_build stage: tests - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true variables: GRADLE_MEMORY_MAX: 6G script: @@ -415,7 +540,9 @@ spotless: check-instrumentation-naming: extends: .gradle_build stage: tests - needs: [ ] + needs: + - job: build_java_profiler_ddprof + optional: true script: - ./gradlew --version - ./gradlew checkInstrumentationNaming @@ -423,7 +550,9 @@ check-instrumentation-naming: config-inversion-linter: extends: .gradle_build stage: tests - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true script: - ./gradlew --version - ./gradlew checkConfigurations @@ -432,7 +561,10 @@ test_published_artifacts: extends: .gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests stage: tests - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" script: @@ -459,7 +591,10 @@ test_published_artifacts: .check_job: extends: .gradle_build - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build stage: tests variables: CACHE_TYPE: "lib" @@ -495,7 +630,9 @@ test_published_artifacts: check_build_src: extends: .check_job - needs: [] + needs: + - job: build_java_profiler_ddprof + optional: true variables: GRADLE_TARGET: ":buildSrc:build" @@ -530,7 +667,10 @@ check_debugger: muzzle: extends: .gradle_build - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests rules: - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' @@ -568,7 +708,10 @@ muzzle: muzzle-dep-report: extends: .gradle_build - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests rules: - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/' @@ -611,7 +754,10 @@ muzzle-dep-report: extends: .gradle_build image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers - needs: [ build_tests ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build_tests stage: tests variables: GRADLE_PARAMS: "-PskipFlakyTests" @@ -919,7 +1065,10 @@ deploy_to_di_backend:manual: deploy_to_maven_central: extends: .gradle_build stage: publish - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" rules: @@ -947,7 +1096,10 @@ deploy_to_maven_central: deploy_snapshot_with_ddprof_snapshot: extends: .gradle_build stage: publish - needs: [ build ] + needs: + - job: build_java_profiler_ddprof + optional: true + - build variables: CACHE_TYPE: "lib" rules: diff --git a/dd-java-agent/agent-bootstrap/build.gradle b/dd-java-agent/agent-bootstrap/build.gradle index de0d326341e..fb8791ee3b5 100644 --- a/dd-java-agent/agent-bootstrap/build.gradle +++ b/dd-java-agent/agent-bootstrap/build.gradle @@ -30,6 +30,7 @@ dependencies { testImplementation project(':dd-java-agent:testing') testImplementation group: 'com.google.guava', name: 'guava-testlib', version: '20.0' + testImplementation libs.bundles.mockito } // Must use Java 11 to build JFR enabled code - there is no JFR in OpenJDK 8 (revisit once JFR in Java 8 is available) diff --git a/dd-java-agent/agent-bootstrap/gradle.lockfile b/dd-java-agent/agent-bootstrap/gradle.lockfile index 0ba47412aa0..f1bc3746f3e 100644 --- a/dd-java-agent/agent-bootstrap/gradle.lockfile +++ b/dd-java-agent/agent-bootstrap/gradle.lockfile @@ -93,7 +93,8 @@ org.junit.platform:junit-platform-suite-api:1.14.1=jmhRuntimeClasspath,testRunti org.junit.platform:junit-platform-suite-commons:1.14.1=jmhRuntimeClasspath,testRuntimeClasspath org.junit:junit-bom:5.14.0=spotbugs org.junit:junit-bom:5.14.1=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.mockito:mockito-core:4.4.0=jmhRuntimeClasspath,testRuntimeClasspath +org.mockito:mockito-core:4.4.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-junit-jupiter:4.4.0=testCompileClasspath,testRuntimeClasspath org.objenesis:objenesis:3.3=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath org.openjdk.jmh:jmh-core:1.37=jmh,jmhCompileClasspath,jmhRuntimeClasspath org.openjdk.jmh:jmh-generator-asm:1.37=jmh,jmhCompileClasspath,jmhRuntimeClasspath diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java new file mode 100644 index 00000000000..e1ef9dc32c6 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/LockSupportHelper.java @@ -0,0 +1,102 @@ +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Helper for profiling {@code LockSupport.park*} intervals from bootstrap classes. + * + *

Known limitation: spurious-wakeup mis-attribution

+ * + *

{@link #UNPARKING_SPAN} pairs an {@code unpark(t)} caller's span id with the next {@code + * park*} return on thread {@code t}. {@code LockSupport.park*} is documented to be allowed to + * return spuriously, in which case the parked thread re-parks without ever consuming the map entry. + * A subsequent, unrelated {@code park*} call on the same thread will then drain the stale entry and + * incorrectly attribute the unblocking span to a {@code TaskBlock} it did not cause. + * + *

We accept this residual race because the correct fix (per-park sequence numbers carried + * through {@code ProfilerContext} and matched on entry) is disproportionate to the rarity of + * spurious wake-ups on the JDKs we target, and because the worst-case impact is a single + * mis-attributed {@code TaskBlock} event per occurrence. + */ +public final class LockSupportHelper { + /** + * Maps target thread id to the span ID of the thread that called {@code unpark()} on it. Keyed by + * {@link Thread#getId()} rather than {@code Thread} so terminated threads can be GC'd; the map + * otherwise lives for the JVM lifetime. + */ + public static final ConcurrentHashMap UNPARKING_SPAN = new ConcurrentHashMap<>(); + + private LockSupportHelper() {} + + /** Captured state for a {@code LockSupport.park*} interval. */ + public static final class ParkState { + public final ProfilingContextIntegration profiling; + public final long blockerHash; + public final long spanId; + public final long rootSpanId; + + public ParkState( + ProfilingContextIntegration profiling, long blockerHash, long spanId, long rootSpanId) { + this.profiling = profiling; + this.blockerHash = blockerHash; + this.spanId = spanId; + this.rootSpanId = rootSpanId; + } + } + + public static ParkState captureState(Object blocker) { + return captureState(blocker, AgentTracer.get().getProfilingContext(), AgentTracer.activeSpan()); + } + + public static ParkState captureState( + Object blocker, ProfilingContextIntegration profiling, AgentSpan span) { + if (profiling == null) { + return null; + } + // Always call parkEnter for signal suppression, even without an active span. + // spanId/rootSpanId = 0 when no active span, and native TaskBlock eligibility filters out + // zero-span intervals at exit. + long spanId = 0L; + long rootSpanId = 0L; + ProfilerContext ctx = ProfilerContexts.of(span); + if (ctx != null) { + spanId = ctx.getSpanId(); + rootSpanId = ctx.getRootSpanId(); + } + profiling.parkEnter(spanId, rootSpanId); + long blockerHash = blocker != null ? System.identityHashCode(blocker) : 0L; + return new ParkState(profiling, blockerHash, spanId, rootSpanId); + } + + public static void finish(ParkState state) { + // Always drain the map entry before any early return. If we returned first, a stale + // unblocking-span ID placed by a prior unpark() would persist and be incorrectly + // attributed to the next TaskBlock event emitted on this thread. + Long unblockingSpanId = UNPARKING_SPAN.remove(Thread.currentThread().getId()); + finish(state, unblockingSpanId != null ? unblockingSpanId : 0L); + } + + public static void finish(ParkState state, long unblockingSpanId) { + if (state == null) { + return; + } + // parkExit() clears native parked state and records an eligible TaskBlock using the entry + // tick saved by parkEnter(). + state.profiling.parkExit(state.blockerHash, unblockingSpanId); + } + + public static void recordUnpark(Thread thread) { + if (thread == null) { + return; + } + ProfilerContext ctx = ProfilerContexts.of(AgentTracer.activeSpan()); + if (ctx == null) { + return; + } + UNPARKING_SPAN.put(thread.getId(), ctx.getSpanId()); + } +} diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/ProfilerContexts.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/ProfilerContexts.java new file mode 100644 index 00000000000..9a20dc0a612 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/ProfilerContexts.java @@ -0,0 +1,24 @@ +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; + +/** Internal helpers shared by Java-level blocking instrumentation. */ +final class ProfilerContexts { + private ProfilerContexts() {} + + /** + * Returns the {@link ProfilerContext} for the given span, or {@code null} if the span is absent + * or its context does not implement {@code ProfilerContext}. Profiling-aware instrumentation + * always takes this path before reading span/root-span IDs, so centralising the {@code + * instanceof} check keeps the semantics identical across helpers. + */ + static ProfilerContext of(AgentSpan span) { + if (span == null) { + return null; + } + AgentSpanContext context = span.context(); + return context instanceof ProfilerContext ? (ProfilerContext) context : null; + } +} diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper.java new file mode 100644 index 00000000000..c6986ca06b4 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelper.java @@ -0,0 +1,67 @@ +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; + +/** Helper for Java-level instrumentation that emits {@code datadog.TaskBlock} intervals. */ +public final class TaskBlockHelper { + static final long MIN_TASK_BLOCK_NANOS = 1_000_000L; + + private TaskBlockHelper() {} + + /** Captured state for a potential blocking interval. */ + public static final class State { + final ProfilingContextIntegration profiling; + final long startTicks; + final long startNanos; + final long spanId; + final long rootSpanId; + final long blocker; + + State( + ProfilingContextIntegration profiling, + long startTicks, + long startNanos, + long spanId, + long rootSpanId, + long blocker) { + this.profiling = profiling; + this.startTicks = startTicks; + this.startNanos = startNanos; + this.spanId = spanId; + this.rootSpanId = rootSpanId; + this.blocker = blocker; + } + } + + public static State capture(long blocker) { + return capture(blocker, AgentTracer.get().getProfilingContext(), AgentTracer.activeSpan()); + } + + static State capture(long blocker, ProfilingContextIntegration profiling, AgentSpan span) { + if (profiling == null) { + return null; + } + ProfilerContext context = ProfilerContexts.of(span); + if (context == null) { + return null; + } + return new State( + profiling, + profiling.getCurrentTicks(), + System.nanoTime(), + context.getSpanId(), + context.getRootSpanId(), + blocker); + } + + public static void finish(State state) { + if (state == null || System.nanoTime() - state.startNanos < MIN_TASK_BLOCK_NANOS) { + return; + } + state.profiling.recordTaskBlock( + state.startTicks, state.spanId, state.rootSpanId, state.blocker, 0L); + } +} diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperTest.java new file mode 100644 index 00000000000..7e05fea33e3 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TaskBlockHelperTest.java @@ -0,0 +1,118 @@ +package datadog.trace.bootstrap.instrumentation.java.concurrent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import org.junit.jupiter.api.Test; + +class TaskBlockHelperTest { + + private static final long SPAN_ID = 0xDEADBEEFL; + private static final long ROOT_SPAN_ID = 0xCAFEBABEL; + private static final long START_TICKS = 42_000_000L; + private static final long BLOCKER = 1234L; + + private interface ProfilerSpanContext extends AgentSpanContext, ProfilerContext {} + + @Test + void capture_returnsNull_withoutProfilingContext() { + AgentSpan span = mock(AgentSpan.class); + + assertNull(TaskBlockHelper.capture(BLOCKER, null, span)); + } + + @Test + void capture_returnsNull_withoutActiveSpan() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + + assertNull(TaskBlockHelper.capture(BLOCKER, profiling, null)); + } + + @Test + void capture_returnsNull_whenSpanContextIsNotProfilerContext() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + AgentSpan nonProfilerSpan = mock(AgentSpan.class); + AgentSpanContext nonProfilerCtx = mock(AgentSpanContext.class); + when(nonProfilerSpan.context()).thenReturn(nonProfilerCtx); + + assertNull(TaskBlockHelper.capture(BLOCKER, profiling, nonProfilerSpan)); + } + + @Test + void capture_recordsActiveSpanAndEntryTiming() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + AgentSpan span = mock(AgentSpan.class); + ProfilerSpanContext ctx = mock(ProfilerSpanContext.class); + when(span.context()).thenReturn(ctx); + when(ctx.getSpanId()).thenReturn(SPAN_ID); + when(ctx.getRootSpanId()).thenReturn(ROOT_SPAN_ID); + when(profiling.getCurrentTicks()).thenReturn(START_TICKS); + + long before = System.nanoTime(); + TaskBlockHelper.State state = TaskBlockHelper.capture(BLOCKER, profiling, span); + long after = System.nanoTime(); + + assertNotNull(state); + assertEquals(profiling, state.profiling); + assertEquals(START_TICKS, state.startTicks); + assertTrue(state.startNanos >= before, "startNanos should be captured after `before`"); + assertTrue(state.startNanos <= after, "startNanos should be captured before `after`"); + assertEquals(SPAN_ID, state.spanId); + assertEquals(ROOT_SPAN_ID, state.rootSpanId); + assertEquals(BLOCKER, state.blocker); + } + + @Test + void finish_ignoresNullState() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + + TaskBlockHelper.finish(null); + + verifyNoInteractions(profiling); + } + + @Test + void finish_ignoresTooShortIntervals() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + // startNanos far in the future so (now - startNanos) is negative and below threshold + TaskBlockHelper.State state = + new TaskBlockHelper.State( + profiling, + START_TICKS, + System.nanoTime() + 60_000_000_000L, + SPAN_ID, + ROOT_SPAN_ID, + BLOCKER); + + TaskBlockHelper.finish(state); + + verifyNoInteractions(profiling); + } + + @Test + void finish_emitsTaskBlockForEligibleInterval() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + TaskBlockHelper.State state = + new TaskBlockHelper.State( + profiling, + START_TICKS, + System.nanoTime() - 2 * TaskBlockHelper.MIN_TASK_BLOCK_NANOS, + SPAN_ID, + ROOT_SPAN_ID, + BLOCKER); + + TaskBlockHelper.finish(state); + + verify(profiling).recordTaskBlock(START_TICKS, SPAN_ID, ROOT_SPAN_ID, BLOCKER, 0L); + } +} diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index b1e07b08c32..8abf48b769b 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -341,6 +341,17 @@ public void recordTraceRoot(long rootSpanId, String endpoint, String operation) } } + /** Monotonic tick count for TaskBlock and wall-clock off-CPU interval timing. */ + public long getCurrentTicks() { + return profiler.getCurrentTicks(); + } + + int encode(CharSequence constant) { + // java-profiler ContextSetter no longer exposes value encoding. + // Keep API contract by returning "not encoded" (0), which callers already handle. + return 0; + } + public int operationNameOffset() { return offsetOf(OPERATION); } @@ -454,4 +465,24 @@ void recordQueueTimeEvent( } } } + + void recordTaskBlockEvent( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + if (profiler != null) { + long endTicks = profiler.getCurrentTicks(); + profiler.recordTaskBlock(startTicks, endTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + } + + void parkEnter(long spanId, long rootSpanId) { + if (profiler != null) { + profiler.parkEnter(spanId, rootSpanId); + } + } + + void parkExit(long blocker, long unblockingSpanId) { + if (profiler != null) { + profiler.parkExit(blocker, unblockingSpanId); + } + } } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 00a0358d346..67af75a7afd 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -71,11 +71,53 @@ public void onDetach() { } } + @Override + public int encode(CharSequence constant) { + return DDPROF.encode(constant); + } + + @Override + public int encodeOperationName(CharSequence constant) { + if (SPAN_NAME_INDEX >= 0) { + return DDPROF.encode(constant); + } + return 0; + } + + @Override + public int encodeResourceName(CharSequence constant) { + if (RESOURCE_NAME_INDEX >= 0) { + return DDPROF.encode(constant); + } + return 0; + } + @Override public String name() { return "ddprof"; } + @Override + public long getCurrentTicks() { + return DDPROF.getCurrentTicks(); + } + + @Override + public void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) { + DDPROF.recordTaskBlockEvent(startTicks, spanId, rootSpanId, blocker, unblockingSpanId); + } + + @Override + public void parkEnter(long spanId, long rootSpanId) { + DDPROF.parkEnter(spanId, rootSpanId); + } + + @Override + public void parkExit(long blocker, long unblockingSpanId) { + DDPROF.parkExit(blocker, unblockingSpanId); + } + public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index 55d39ba52a0..80754c7937f 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.Properties; import java.util.UUID; +import java.util.concurrent.locks.LockSupport; import java.util.stream.IntStream; import java.util.stream.Stream; import org.junit.jupiter.api.Assumptions; @@ -29,6 +30,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.ItemFilters; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,6 +77,47 @@ void test() throws Exception { } } + @Test + void testTaskBlockBridgeMethodsEmitTaskBlockEvents() throws Exception { + assertDoesNotThrow( + () -> DdprofLibraryLoader.jvmAccess().getReasonNotLoaded(), "Profiler not available"); + DatadogProfiler profiler = DatadogProfiler.newInstance(ConfigProvider.getInstance()); + if (profiler.isActive()) { + log.warn("Datadog profiler is already running. Skipping task-block integration test."); + return; + } + + OngoingRecording recording = profiler.start(); + if (recording == null) { + log.warn("Datadog Profiler is not available. Skipping task-block integration test."); + return; + } + + try { + // Direct bridge path (recordTaskBlock -> JavaProfiler.recordTaskBlock0) + long startTicks = profiler.getCurrentTicks(); + LockSupport.parkNanos(3_000_000L); // > 1ms native threshold + profiler.recordTaskBlockEvent(startTicks, 101L, 202L, 303L, 404L); + + // Park path (parkEnter/parkExit -> JavaProfiler.parkEnter0/parkExit0) + profiler.parkEnter(505L, 606L); + LockSupport.parkNanos(3_000_000L); // > 1ms native threshold + profiler.parkExit(707L, 808L); + + RecordingData data = profiler.stop(recording); + assertNotNull(data); + IItemCollection events = JfrLoaderToolkit.loadEvents(data.getStream()); + long taskBlockCount = + events.apply(ItemFilters.type("datadog.TaskBlock")).stream() + .mapToLong(IItemIterable::getItemCount) + .sum(); + + assertTrue(taskBlockCount > 0, "Expected datadog.TaskBlock events from bridge methods"); + } finally { + recording.stop(); + } + } + @ParameterizedTest @MethodSource("profilingModes") void testStartCmd(boolean cpu, boolean wall, boolean alloc, boolean memleak) throws Exception { diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index 36cedaf6081..c22ff08b5cb 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -57,6 +57,8 @@ 0 java.lang.Runtime # allow context tracking for VirtualThread 0 java.lang.VirtualThread +# allow object-wait profiling instrumentation (JDK 21+) +0 java.lang.Object 0 java.net.http.* 0 java.net.HttpURLConnection 0 java.net.InetAddress @@ -70,6 +72,7 @@ 1 java.util.concurrent.ConcurrentHashMap* 1 java.util.concurrent.atomic.* 1 java.util.concurrent.locks.* +0 java.util.concurrent.locks.LockSupport 0 java.util.logging.* # allow capturing JVM shutdown 0 java.lang.Shutdown diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle new file mode 100644 index 00000000000..f2f91cdc7fd --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/build.gradle @@ -0,0 +1,12 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + coreJdk() + } +} + +dependencies { + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java new file mode 100644 index 00000000000..b836cf2c79f --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/main/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentation.java @@ -0,0 +1,101 @@ +package datadog.trace.instrumentation.locksupport; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isStatic; +import static net.bytebuddy.matcher.ElementMatchers.nameStartsWith; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.java.concurrent.LockSupportHelper; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; +import net.bytebuddy.matcher.ElementMatchers; + +/** + * Instruments {@link java.util.concurrent.locks.LockSupport#park} variants as the Java entry point + * for native parked-state tracking. The native profiler uses this state to suppress wall-clock + * signals while the thread is parked and, when the interval belongs to an active span, to emit a + * replacement {@code datadog.TaskBlock} event on {@code parkExit}. + * + *

Also instruments {@link java.util.concurrent.locks.LockSupport#unpark} to capture the span ID + * of the unblocking thread, which is then recorded in the native TaskBlock event. + * + *

{@code parkEnter} runs even without an active span (span id 0) so the native wall-clock + * precheck can suppress {@code SIGVTALRM} for the whole park interval. TaskBlock JFR emission is + * gated by the profiler on duration and span context. + */ +@AutoService(InstrumenterModule.class) +public class LockSupportProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + public LockSupportProfilingInstrumentation() { + super("lock-support"); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.util.concurrent.locks.LockSupport"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + parkMethod().and(takesArgument(0, named("java.lang.Object"))), + getClass().getName() + "$ParkWithBlockerAdvice"); + transformer.applyAdvice( + parkMethod().and(ElementMatchers.not(takesArgument(0, named("java.lang.Object")))), + getClass().getName() + "$ParkWithoutBlockerAdvice"); + transformer.applyAdvice( + isMethod() + .and(isStatic()) + .and(ElementMatchers.named("unpark")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))), + getClass().getName() + "$UnparkAdvice"); + } + + private static ElementMatcher.Junction parkMethod() { + return isMethod() + .and(isStatic()) + .and(nameStartsWith("park")) + .and(isDeclaredBy(named("java.util.concurrent.locks.LockSupport"))); + } + + public static final class ParkWithBlockerAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static LockSupportHelper.ParkState before(@Advice.Argument(0) Object blocker) { + return LockSupportHelper.captureState(blocker); + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter LockSupportHelper.ParkState state) { + LockSupportHelper.finish(state); + } + } + + public static final class ParkWithoutBlockerAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static LockSupportHelper.ParkState before() { + return LockSupportHelper.captureState(null); + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter LockSupportHelper.ParkState state) { + LockSupportHelper.finish(state); + } + } + + public static final class UnparkAdvice { + + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before(@Advice.Argument(0) Thread thread) { + LockSupportHelper.recordUnpark(thread); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java new file mode 100644 index 00000000000..dfd0a2b8174 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/lock-support/src/test/java/datadog/trace/instrumentation/locksupport/LockSupportProfilingInstrumentationTest.java @@ -0,0 +1,265 @@ +package datadog.trace.instrumentation.locksupport; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import datadog.trace.bootstrap.instrumentation.java.concurrent.LockSupportHelper; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LockSupportProfilingInstrumentation}. + * + *

These tests exercise the {@link LockSupportHelper} map directly, verifying the mechanism used + * to communicate the unblocking span ID from {@code UnparkAdvice} to {@code ParkAdvice}. + */ +class LockSupportProfilingInstrumentationTest { + + private static final long SPAN_ID = 1234L; + private static final long ROOT_SPAN_ID = 5678L; + + private interface ProfilerSpanContext extends AgentSpanContext, ProfilerContext {} + + @BeforeEach + void clearState() { + LockSupportHelper.UNPARKING_SPAN.clear(); + } + + @AfterEach + void cleanupState() { + LockSupportHelper.UNPARKING_SPAN.clear(); + } + + // ------------------------------------------------------------------------- + // State map — basic contract + // ------------------------------------------------------------------------- + + @Test + void state_put_and_remove() { + long tid = Thread.currentThread().getId(); + long spanId = 12345L; + + LockSupportHelper.UNPARKING_SPAN.put(tid, spanId); + Long retrieved = LockSupportHelper.UNPARKING_SPAN.remove(tid); + + assertNotNull(retrieved); + assertEquals(spanId, (long) retrieved); + // After removal the entry should be gone + assertNull(LockSupportHelper.UNPARKING_SPAN.get(tid)); + } + + @Test + void state_remove_returns_null_when_absent() { + Thread t = new Thread(() -> {}); + assertNull(LockSupportHelper.UNPARKING_SPAN.remove(t.getId())); + } + + @Test + void state_is_initially_empty() { + assertTrue(LockSupportHelper.UNPARKING_SPAN.isEmpty()); + } + + // ------------------------------------------------------------------------- + // Multithreaded: unpark thread populates map, parked thread reads it + // ------------------------------------------------------------------------- + + /** + * Simulates the UnparkAdvice → ParkAdvice handoff: + * + *

    + *
  1. Thread A (the "parked" thread) blocks on a latch. + *
  2. Thread B (the "unparking" thread) places its span ID in {@code State.UNPARKING_SPAN} for + * Thread A and then releases the latch. + *
  3. Thread A wakes up, reads and removes the span ID from the map. + *
+ */ + @Test + void unparking_spanId_is_visible_to_parked_thread() throws InterruptedException { + long unparkingSpanId = 99887766L; + + CountDownLatch ready = new CountDownLatch(1); + CountDownLatch go = new CountDownLatch(1); + AtomicLong capturedSpanId = new AtomicLong(-1L); + AtomicReference parkedThreadRef = new AtomicReference<>(); + + Thread parkedThread = + new Thread( + () -> { + parkedThreadRef.set(Thread.currentThread()); + ready.countDown(); + try { + go.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Simulate what ParkAdvice.after does: read and remove unblocking span id + Long unblockingId = + LockSupportHelper.UNPARKING_SPAN.remove(Thread.currentThread().getId()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + + parkedThread.start(); + ready.await(); // wait for parked thread to register itself + + // Simulate what UnparkAdvice.before does: record unparking span id + LockSupportHelper.UNPARKING_SPAN.put(parkedThread.getId(), unparkingSpanId); + go.countDown(); // unblock parked thread + + parkedThread.join(2_000); + assertFalse(parkedThread.isAlive(), "Test thread did not finish in time"); + assertEquals( + unparkingSpanId, + capturedSpanId.get(), + "Parked thread should have read the unblocking span id placed by unparking thread"); + } + + /** + * Verifies that if no entry exists for the parked thread (i.e. the thread was unblocked by a + * non-traced thread), the {@code remove} returns {@code null} and the code falls back to 0. + */ + @Test + void no_unparking_entry_yields_zero() throws InterruptedException { + AtomicLong capturedSpanId = new AtomicLong(-1L); + + Thread parkedThread = + new Thread( + () -> { + Long unblockingId = + LockSupportHelper.UNPARKING_SPAN.remove(Thread.currentThread().getId()); + capturedSpanId.set(unblockingId != null ? unblockingId : 0L); + }); + parkedThread.start(); + parkedThread.join(2_000); + + assertEquals( + 0L, capturedSpanId.get(), "Should fall back to 0 when no unparking span id is recorded"); + } + + // ------------------------------------------------------------------------- + // ParkAdvice.after — null state is a no-op + // ------------------------------------------------------------------------- + + /** + * When {@code ParkAdvice.before} returns {@code null} (profiler not active or no active span), + * {@code ParkAdvice.after} must not throw and must not leave entries in {@code UNPARKING_SPAN}. + * It does call {@code remove(currentThread)}, but on an empty map that is a no-op. + */ + @Test + void parkAdvice_after_null_state_isNoOp() { + LockSupportHelper.finish(null); + assertTrue(LockSupportHelper.UNPARKING_SPAN.isEmpty()); + } + + @Test + void parkAdvice_captureState_nullProfiling_returnsNull() { + AgentSpan span = mock(AgentSpan.class); + + assertNull(LockSupportHelper.captureState(new Object(), null, span)); + } + + @Test + void parkAdvice_captureState_spanless_callsParkEnterWithZeroIds() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + + LockSupportHelper.ParkState state = + LockSupportHelper.captureState(new Object(), profiling, null); + + assertNotNull(state); + assertEquals(0L, state.spanId); + assertEquals(0L, state.rootSpanId); + verify(profiling).parkEnter(0L, 0L); + } + + @Test + void parkAdvice_captureState_activeSpan_callsParkEnterWithSpanIds() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + AgentSpan span = mock(AgentSpan.class); + ProfilerSpanContext context = mock(ProfilerSpanContext.class); + Object blocker = new Object(); + when(span.context()).thenReturn(context); + when(context.getSpanId()).thenReturn(SPAN_ID); + when(context.getRootSpanId()).thenReturn(ROOT_SPAN_ID); + + LockSupportHelper.ParkState state = LockSupportHelper.captureState(blocker, profiling, span); + + assertNotNull(state); + assertEquals(System.identityHashCode(blocker), state.blockerHash); + assertEquals(SPAN_ID, state.spanId); + assertEquals(ROOT_SPAN_ID, state.rootSpanId); + verify(profiling).parkEnter(SPAN_ID, ROOT_SPAN_ID); + } + + @Test + void parkAdvice_finish_callsOriginalProfilingContext() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + LockSupportHelper.ParkState state = + new LockSupportHelper.ParkState(profiling, 42L, SPAN_ID, ROOT_SPAN_ID); + + LockSupportHelper.finish(state, 99L); + + verify(profiling).parkExit(42L, 99L); + } + + @Test + void parkAdvice_finish_nullState_doesNotTouchProfiling() { + ProfilingContextIntegration profiling = mock(ProfilingContextIntegration.class); + + LockSupportHelper.finish(null, 99L); + + verifyNoInteractions(profiling); + } + + /** + * Regression test for stale-entry misattribution. + * + *

If {@code unpark(t)} is called (inserting an entry into {@code UNPARKING_SPAN}) and thread + * {@code t} then parks without an active span ({@code state == null}), the entry must still be + * drained. Without the fix, it would linger and be incorrectly attributed to the next {@code + * TaskBlock} emitted on that thread. + */ + @Test + void stale_entry_is_drained_when_park_fires_without_active_span() { + long tid = Thread.currentThread().getId(); + LockSupportHelper.UNPARKING_SPAN.put(tid, 99L); + + // Simulate park() returning with no active span (state == null) + LockSupportHelper.finish(null); + + assertNull( + LockSupportHelper.UNPARKING_SPAN.get(tid), + "Stale UNPARKING_SPAN entry must be drained even when state is null"); + } + + /** + * If multiple unpark calls race for the same parked thread, the latest span ID should be consumed + * and the entry must still be drained exactly once by ParkAdvice.after(). + */ + @Test + void latest_unparking_span_wins_and_entry_is_drained() { + long tid = Thread.currentThread().getId(); + LockSupportHelper.UNPARKING_SPAN.put(tid, 101L); + LockSupportHelper.UNPARKING_SPAN.put(tid, 202L); + + Long consumed = LockSupportHelper.UNPARKING_SPAN.remove(tid); + assertNotNull(consumed); + assertEquals(202L, consumed.longValue()); + assertNull( + LockSupportHelper.UNPARKING_SPAN.get(tid), "Entry must be removed after consumption"); + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle b/dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle new file mode 100644 index 00000000000..d61e8bd6155 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait/build.gradle @@ -0,0 +1,16 @@ +apply from: "$rootDir/gradle/java.gradle" + +testJvmConstraints { + minJavaVersion = JavaVersion.VERSION_21 +} + +muzzle { + pass { + coreJdk('21') + } +} + +dependencies { + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java new file mode 100644 index 00000000000..b13299c61cb --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/main/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentation.java @@ -0,0 +1,79 @@ +package datadog.trace.instrumentation.objectwait; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import datadog.environment.JavaVirtualMachine; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.java.concurrent.TaskBlockHelper; +import net.bytebuddy.asm.Advice; + +/** + * Instruments {@link Object#wait(long)} in JDK 21+ to emit {@code datadog.TaskBlock} JFR events. + * + *

In JDK 21+, {@code wait(long)} is a pure-Java wrapper around the native {@code wait0(long)}, + * so ByteBuddy can add advice to it. In JDK 8-20 the method is declared {@code native} and is not + * instrumented by this class (Approach 1 osThreadState precheck already suppresses SIGVTALRM for + * threads in OBJECT_WAIT state on all JDK versions). + * + *

Only {@code wait(long)} is instrumented: {@code wait()} delegates to {@code wait(0L)} and + * {@code wait(long, int)} delegates to {@code wait(long)}, so all wait variants are covered. + * + *

{@code unblockingSpanId} is always 0 because {@code notify()} and {@code notifyAll()} remain + * {@code native} in JDK 21+ and the notifying thread cannot be identified via BCI. + */ +@AutoService(InstrumenterModule.class) +public class ObjectWaitProfilingInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + public ObjectWaitProfilingInstrumentation() { + super("object-wait"); + } + + @Override + public boolean isEnabled() { + return JavaVirtualMachine.isJavaVersionAtLeast(21) && super.isEnabled(); + } + + @Override + public String[] knownMatchingTypes() { + return new String[] {"java.lang.Object"}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(named("wait")) + .and(takesArguments(1)) + .and(takesArgument(0, long.class)) + .and(isDeclaredBy(named("java.lang.Object"))), + getClass().getName() + "$WaitAdvice"); + } + + public static final class WaitAdvice { + + /* + * IMPORTANT: advice bodies must only reference classes visible to the bootstrap class + * loader (here, TaskBlockHelper in agent-bootstrap). ByteBuddy inlines the bytecode of + * these methods into java.lang.Object#wait(long), but any static call back to a helper + * on this advice class itself would be left as INVOKESTATIC against a class living in + * the agent class loader, producing NoClassDefFoundError at every wait() site. Do not + * extract helpers onto this class. + */ + @Advice.OnMethodEnter(suppress = Throwable.class) + public static TaskBlockHelper.State before(@Advice.This Object monitor) { + return TaskBlockHelper.capture(System.identityHashCode(monitor)); + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void after(@Advice.Enter TaskBlockHelper.State state) { + TaskBlockHelper.finish(state); + } + } +} diff --git a/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java new file mode 100644 index 00000000000..b556e8bc9e9 --- /dev/null +++ b/dd-java-agent/instrumentation/datadog/profiling/object-wait/src/test/java/datadog/trace/instrumentation/objectwait/ObjectWaitProfilingInstrumentationTest.java @@ -0,0 +1,63 @@ +package datadog.trace.instrumentation.objectwait; + +import static org.mockito.Mockito.verify; + +import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; +import datadog.trace.bootstrap.instrumentation.java.concurrent.TaskBlockHelper; +import datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation.WaitAdvice; +import java.lang.reflect.Constructor; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Unit tests for {@link ObjectWaitProfilingInstrumentation}. + * + *

Detailed interval eligibility is covered by {@code TaskBlockHelperTest}; this class verifies + * that Object.wait advice delegates to the shared Java-level TaskBlock helper. + */ +@ExtendWith(MockitoExtension.class) +class ObjectWaitProfilingInstrumentationTest { + + private static final long SPAN_ID = 0xDEADBEEFL; + private static final long ROOT_SPAN_ID = 0xCAFEBABEL; + private static final long START_TICKS = 42_000_000L; + private static final long BLOCKER = 1234L; + + @Mock private ProfilingContextIntegration profiling; + + @Test + void after_nullState_doesNotThrow() { + WaitAdvice.after(null); + } + + @Test + void after_eligibleState_emitsTaskBlockWithZeroUnblockingSpanId() throws Exception { + TaskBlockHelper.State state = newState(System.nanoTime() - 2 * taskBlockThresholdNanos()); + + WaitAdvice.after(state); + + verify(profiling).recordTaskBlock(START_TICKS, SPAN_ID, ROOT_SPAN_ID, BLOCKER, 0L); + } + + private TaskBlockHelper.State newState(long startNanos) throws Exception { + Constructor constructor = + TaskBlockHelper.State.class.getDeclaredConstructor( + ProfilingContextIntegration.class, + long.class, + long.class, + long.class, + long.class, + long.class); + constructor.setAccessible(true); + return constructor.newInstance( + profiling, START_TICKS, startNanos, SPAN_ID, ROOT_SPAN_ID, BLOCKER); + } + + private static long taskBlockThresholdNanos() throws Exception { + java.lang.reflect.Field field = TaskBlockHelper.class.getDeclaredField("MIN_TASK_BLOCK_NANOS"); + field.setAccessible(true); + return field.getLong(null); + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java new file mode 100644 index 00000000000..88a61ec1c67 --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/LockSupportTaskBlockProfilingTest.java @@ -0,0 +1,392 @@ +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.agentShadowJar; +import static datadog.smoketest.SmokeTestUtils.buildDirectory; +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.SmokeTestUtils.javaPath; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; +import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; + +import datadog.trace.api.config.ProfilingConfig; +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +@DisabledOnJ9 +final class LockSupportTaskBlockProfilingTest { + private static final byte[] JFR_MAGIC = new byte[] {'F', 'L', 'R', 0}; + private static final IAttribute SPAN_ID = attr("spanId", "spanId", "spanId", NUMBER); + private static final IAttribute LOCAL_ROOT_SPAN_ID = + attr("localRootSpanId", "localRootSpanId", "localRootSpanId", NUMBER); + private static final IAttribute BLOCKER = + attr("blocker", "blocker", "blocker", NUMBER); + private static final IAttribute UNBLOCKING_SPAN_ID = + attr("unblockingSpanId", "unblockingSpanId", "unblockingSpanId", NUMBER); + private static final IAttribute TASK_BLOCK_EMITTED = + attr("numTaskBlockEmitted", "numTaskBlockEmitted", "numTaskBlockEmitted", NUMBER); + private static final IAttribute TASK_BLOCK_SKIPPED_SPAN_ZERO = + attr( + "numTaskBlockSkippedSpanZero", + "numTaskBlockSkippedSpanZero", + "numTaskBlockSkippedSpanZero", + NUMBER); + private static final IAttribute TASK_BLOCK_SKIPPED_TOO_SHORT = + attr( + "numTaskBlockSkippedTooShort", + "numTaskBlockSkippedTooShort", + "numTaskBlockSkippedTooShort", + NUMBER); + private static final IAttribute OPERATION = + attr("_dd.trace.operation", "_dd.trace.operation", "_dd.trace.operation", PLAIN_TEXT); + private static final Path LOG_FILE_BASE = + Paths.get( + buildDirectory(), + "reports", + "testProcess." + LockSupportTaskBlockProfilingTest.class.getName()); + + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + Files.createDirectories(LOG_FILE_BASE); + logFilePath = + LOG_FILE_BASE.resolve( + testInfo.getTestMethod().map(method -> method.getName()).orElse("lockSupport") + + ".log"); + dumpDir = Files.createTempDirectory("dd-profiler-locksupport-"); + } + + @AfterEach + void tearDown() throws IOException { + if (dumpDir != null && Files.exists(dumpDir)) { + Files.walk(dumpDir).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + + @Test + @DisplayName("LockSupport parks emit span-attributed TaskBlock events") + void lockSupportParksEmitTaskBlockEvents() throws Exception { + Process targetProcess = createProcessBuilder().start(); + + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = loadStats(); + assertTrue(stats.taskBlockCount > 0, "Expected datadog.TaskBlock events"); + assertTrue(stats.taskBlockEmitted > 0, "Expected numTaskBlockEmitted counter"); + assertTrue(stats.taskBlockSkippedSpanZero > 0, "Expected spanless parks to be skipped"); + assertTrue(stats.taskBlockSkippedTooShort > 0, "Expected short parks to be skipped"); + assertTrue(stats.taskBlocksWithNonZeroBlocker > 0, "Expected blocker identity to be recorded"); + assertTrue(stats.taskBlocksWithUnblockingSpan > 0, "Expected unblocking span to be recorded"); + assertFalse(stats.hasZeroSpanId, "TaskBlock events must have non-zero spanId"); + assertFalse( + stats.hasZeroLocalRootSpanId, "TaskBlock events must have non-zero localRootSpanId"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertTrue( + stats.hasExpectedOperation, + "Expected TaskBlock events to include LockSupport span operation names"); + assertFalse(logHasLockSupportInstrumentationError(), "LockSupport instrumentation failed"); + } + + private ProcessBuilder createProcessBuilder() { + String templateOverride = + LockSupportTaskBlockProfilingTest.class + .getClassLoader() + .getResource("overrides.jfp") + .getFile(); + List command = + new ArrayList<>( + Arrays.asList( + javaPath(), + "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), + "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), + "-javaagent:" + agentShadowJar(), + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Ddd.service.name=smoke-test-locksupport-taskblock", + "-Ddd.env=smoketest", + "-Ddd.version=99", + "-Ddd.profiling.enabled=true", + "-Ddd.profiling.ddprof.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_AUXILIARY_TYPE + "=async", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED + "=true", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_INTERVAL + "=10ms", + "-Ddd.profiling.agentless=false", + "-Ddd.profiling.start-delay=0", + "-Ddd." + ProfilingConfig.PROFILING_START_FORCE_FIRST + "=true", + "-Ddd.profiling.upload.period=1", + "-Ddd.profiling.hotspots.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES_SPAN_NAME_ENABLED + "=true", + "-Ddd.profiling.debug.dump_path=" + dumpDir, + "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", + "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug", + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UnlockCommercialFeatures", + "-XX:+FlightRecorder", + "-Ddd." + ProfilingConfig.PROFILING_TEMPLATE_OVERRIDE_FILE + "=" + templateOverride, + "-cp", + System.getProperty("java.class.path"), + LockSupportTaskBlockForkedApp.class.getName())); + if (System.getenv("TEST_LIBASYNC") != null) { + command.add( + command.size() - 3, + "-Ddd." + + ProfilingConfig.PROFILING_DATADOG_PROFILER_LIBPATH + + "=" + + System.getenv("TEST_LIBASYNC")); + } + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(new File(buildDirectory())); + processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")); + processBuilder.redirectErrorStream(true); + processBuilder.redirectOutput(ProcessBuilder.Redirect.to(logFilePath.toFile())); + return processBuilder; + } + + private JfrStats loadStats() throws Exception { + JfrStats stats = new JfrStats(); + List jfrFiles; + try (java.util.stream.Stream files = Files.walk(dumpDir)) { + jfrFiles = + files + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".jfr")) + .collect(Collectors.toList()); + } + for (Path jfrFile : jfrFiles) { + stats.add(loadEvents(jfrFile)); + } + return stats; + } + + private IItemCollection loadEvents(Path path) { + try { + return JfrLoaderToolkit.loadEvents(extractLastJfrStream(path).toFile()); + } catch (Exception e) { + throw new RuntimeException("Failed to load JFR " + path, e); + } + } + + private Path extractLastJfrStream(Path path) throws IOException { + byte[] data = Files.readAllBytes(path); + int lastMagic = lastIndexOf(data, JFR_MAGIC); + if (lastMagic <= 0) { + return path; + } + + Path extracted = dumpDir.resolve(path.getFileName() + ".ddprof.jfr"); + Files.write(extracted, Arrays.copyOfRange(data, lastMagic, data.length)); + return extracted; + } + + private static int lastIndexOf(byte[] data, byte[] needle) { + for (int i = data.length - needle.length; i >= 0; i--) { + boolean match = true; + for (int j = 0; j < needle.length; j++) { + if (data[i + j] != needle[j]) { + match = false; + break; + } + } + if (match) { + return i; + } + } + return -1; + } + + private boolean logHasLockSupportInstrumentationError() throws IOException { + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + return log.contains("NoClassDefFoundError") + || log.contains( + "Failed to handle exception in instrumentation for java.util.concurrent.locks.LockSupport"); + } + + public static final class LockSupportTaskBlockForkedApp { + private static final int PARK_ITERATIONS = 20; + private static final long LONG_PARK_NANOS = TimeUnit.MILLISECONDS.toNanos(50); + private static final long SHORT_PARK_NANOS = 1L; + private static final Object BLOCKER = new Object(); + + public static void main(String[] args) throws Exception { + LockSupportTaskBlockForkedApp app = new LockSupportTaskBlockForkedApp(GlobalTracer.get()); + app.runActiveSpanParks(); + app.runSpanlessParks(); + app.runTooShortParks(); + for (int i = 0; i < 5; i++) { + app.runUnparkAttribution(); + } + Thread.sleep(1500); + } + + private final Tracer tracer; + + private LockSupportTaskBlockForkedApp(Tracer tracer) { + this.tracer = tracer; + } + + private void runActiveSpanParks() { + for (int i = 0; i < PARK_ITERATIONS; i++) { + Span span = tracer.buildSpan("locksupport.active").start(); + try (Scope scope = tracer.activateSpan(span)) { + LockSupport.parkNanos(BLOCKER, LONG_PARK_NANOS); + } finally { + span.finish(); + } + } + } + + private void runSpanlessParks() { + for (int i = 0; i < PARK_ITERATIONS; i++) { + LockSupport.parkNanos(BLOCKER, LONG_PARK_NANOS); + } + } + + private void runTooShortParks() { + for (int i = 0; i < PARK_ITERATIONS; i++) { + Span span = tracer.buildSpan("locksupport.too-short").start(); + try (Scope scope = tracer.activateSpan(span)) { + LockSupport.parkNanos(BLOCKER, SHORT_PARK_NANOS); + } finally { + span.finish(); + } + } + } + + private void runUnparkAttribution() throws Exception { + CountDownLatch parkedThreadReady = new CountDownLatch(1); + Thread parkedThread = + new Thread( + () -> { + Span span = tracer.buildSpan("locksupport.unpark.parked").start(); + try (Scope scope = tracer.activateSpan(span)) { + parkedThreadReady.countDown(); + LockSupport.parkNanos(BLOCKER, TimeUnit.SECONDS.toNanos(5)); + } finally { + span.finish(); + } + }, + "locksupport-taskblock-parked"); + + parkedThread.start(); + parkedThreadReady.await(); + Thread.sleep(50); + + Span unparkingSpan = tracer.buildSpan("locksupport.unpark.unparker").start(); + try (Scope scope = tracer.activateSpan(unparkingSpan)) { + LockSupport.unpark(parkedThread); + } finally { + unparkingSpan.finish(); + } + + parkedThread.join(TimeUnit.SECONDS.toMillis(5)); + if (parkedThread.isAlive()) { + throw new IllegalStateException("Parked thread did not finish"); + } + } + } + + private static final class JfrStats { + private long taskBlockCount; + private long taskBlockEmitted; + private long taskBlockSkippedSpanZero; + private long taskBlockSkippedTooShort; + private long taskBlocksWithNonZeroBlocker; + private long taskBlocksWithUnblockingSpan; + private boolean hasZeroSpanId; + private boolean hasZeroLocalRootSpanId; + private boolean hasMissingEventThread; + private boolean hasExpectedOperation; + + private void add(IItemCollection events) { + addTaskBlocks(events); + addWallClockEpochs(events); + } + + private void addTaskBlocks(IItemCollection events) { + IItemCollection taskBlocks = events.apply(ItemFilters.type("datadog.TaskBlock")); + for (IItemIterable items : taskBlocks) { + IMemberAccessor spanIdAccessor = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor localRootSpanIdAccessor = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blockerAccessor = BLOCKER.getAccessor(items.getType()); + IMemberAccessor unblockingSpanIdAccessor = + UNBLOCKING_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor eventThreadAccessor = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + IMemberAccessor operationAccessor = OPERATION.getAccessor(items.getType()); + for (IItem item : items) { + taskBlockCount++; + long spanId = spanIdAccessor.getMember(item).longValue(); + long localRootSpanId = localRootSpanIdAccessor.getMember(item).longValue(); + long blocker = blockerAccessor.getMember(item).longValue(); + long unblockingSpanId = unblockingSpanIdAccessor.getMember(item).longValue(); + String eventThread = + eventThreadAccessor == null ? null : eventThreadAccessor.getMember(item); + String operation = operationAccessor == null ? null : operationAccessor.getMember(item); + hasZeroSpanId |= spanId == 0; + hasZeroLocalRootSpanId |= localRootSpanId == 0; + hasMissingEventThread |= eventThread == null || eventThread.isEmpty(); + hasExpectedOperation |= + "locksupport.active".equals(operation) + || "locksupport.unpark.parked".equals(operation); + if (blocker != 0) { + taskBlocksWithNonZeroBlocker++; + } + if (unblockingSpanId != 0) { + taskBlocksWithUnblockingSpan++; + } + } + } + } + + private void addWallClockEpochs(IItemCollection events) { + IItemCollection epochs = events.apply(ItemFilters.type("datadog.WallClockSamplingEpoch")); + for (IItemIterable items : epochs) { + IMemberAccessor emittedAccessor = + TASK_BLOCK_EMITTED.getAccessor(items.getType()); + IMemberAccessor spanZeroAccessor = + TASK_BLOCK_SKIPPED_SPAN_ZERO.getAccessor(items.getType()); + IMemberAccessor tooShortAccessor = + TASK_BLOCK_SKIPPED_TOO_SHORT.getAccessor(items.getType()); + for (IItem item : items) { + taskBlockEmitted += emittedAccessor.getMember(item).longValue(); + taskBlockSkippedSpanZero += spanZeroAccessor.getMember(item).longValue(); + taskBlockSkippedTooShort += tooShortAccessor.getMember(item).longValue(); + } + } + } + } +} diff --git a/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java new file mode 100644 index 00000000000..fee3d9d7c0e --- /dev/null +++ b/dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/ObjectWaitTaskBlockProfilingTest.java @@ -0,0 +1,359 @@ +package datadog.smoketest; + +import static datadog.smoketest.SmokeTestUtils.agentShadowJar; +import static datadog.smoketest.SmokeTestUtils.buildDirectory; +import static datadog.smoketest.SmokeTestUtils.checkProcessSuccessfullyEnd; +import static datadog.smoketest.SmokeTestUtils.javaPath; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openjdk.jmc.common.item.Attribute.attr; +import static org.openjdk.jmc.common.unit.UnitLookup.NUMBER; +import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; + +import datadog.environment.JavaVirtualMachine; +import datadog.trace.api.config.ProfilingConfig; +import io.opentracing.Scope; +import io.opentracing.Span; +import io.opentracing.Tracer; +import io.opentracing.util.GlobalTracer; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +/** + * Smoke test for {@link + * datadog.trace.instrumentation.objectwait.ObjectWaitProfilingInstrumentation}. Object.wait is only + * instrumented on JDK 21+ (pre-21 the method is native, and JVMTI MonitorWait callbacks cover that + * path natively). This test is therefore gated on JDK 21+ and guards against regressions such as + * {@code java.lang.Object} being silently excluded from the global ignore trie. + */ +@DisabledOnJ9 +final class ObjectWaitTaskBlockProfilingTest { + private static final byte[] JFR_MAGIC = new byte[] {'F', 'L', 'R', 0}; + private static final IAttribute SPAN_ID = attr("spanId", "spanId", "spanId", NUMBER); + private static final IAttribute LOCAL_ROOT_SPAN_ID = + attr("localRootSpanId", "localRootSpanId", "localRootSpanId", NUMBER); + private static final IAttribute BLOCKER = + attr("blocker", "blocker", "blocker", NUMBER); + private static final IAttribute UNBLOCKING_SPAN_ID = + attr("unblockingSpanId", "unblockingSpanId", "unblockingSpanId", NUMBER); + private static final IAttribute TASK_BLOCK_EMITTED = + attr("numTaskBlockEmitted", "numTaskBlockEmitted", "numTaskBlockEmitted", NUMBER); + private static final IAttribute OPERATION = + attr("_dd.trace.operation", "_dd.trace.operation", "_dd.trace.operation", PLAIN_TEXT); + private static final Path LOG_FILE_BASE = + Paths.get( + buildDirectory(), + "reports", + "testProcess." + ObjectWaitTaskBlockProfilingTest.class.getName()); + + private Path dumpDir; + private Path logFilePath; + + @BeforeEach + void setup(TestInfo testInfo) throws IOException { + // Object.wait advice only applies on JDK 21+ (wait(long) is native on JDK 8-20). + Assumptions.assumeTrue( + JavaVirtualMachine.isJavaVersionAtLeast(21), + "Object.wait instrumentation requires JDK 21+"); + Files.createDirectories(LOG_FILE_BASE); + logFilePath = + LOG_FILE_BASE.resolve( + testInfo.getTestMethod().map(method -> method.getName()).orElse("objectWait") + ".log"); + dumpDir = Files.createTempDirectory("dd-profiler-objectwait-"); + } + + @AfterEach + void tearDown() throws IOException { + if (dumpDir != null && Files.exists(dumpDir)) { + Files.walk(dumpDir).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + + @Test + @DisplayName("Object.wait emits span-attributed TaskBlock events on JDK 21+") + void objectWaitsEmitTaskBlockEvents() throws Exception { + Process targetProcess = createProcessBuilder().start(); + + checkProcessSuccessfullyEnd(targetProcess, logFilePath); + + JfrStats stats = loadStats(); + assertTrue(stats.taskBlockCount > 0, "Expected datadog.TaskBlock events"); + assertTrue(stats.taskBlockEmitted > 0, "Expected numTaskBlockEmitted counter"); + assertTrue(stats.taskBlocksWithNonZeroBlocker > 0, "Expected monitor identity to be recorded"); + assertFalse(stats.hasZeroSpanId, "TaskBlock events must have non-zero spanId"); + assertFalse( + stats.hasZeroLocalRootSpanId, "TaskBlock events must have non-zero localRootSpanId"); + assertFalse(stats.hasMissingEventThread, "TaskBlock events must resolve Event Thread"); + assertTrue( + stats.hasExpectedOperation, + "Expected TaskBlock events to include the objectwait.active span operation name"); + // notify/notifyAll remain native on JDK 21+, so the unblocking thread can never be + // identified via BCI; every Object.wait TaskBlock must report unblockingSpanId == 0. + assertFalse( + stats.hasNonZeroUnblockingSpanId, + "Object.wait TaskBlocks must report unblockingSpanId == 0 (notify is still native)"); + assertFalse(logHasObjectWaitInstrumentationError(), "Object.wait instrumentation failed"); + } + + private ProcessBuilder createProcessBuilder() { + String templateOverride = + ObjectWaitTaskBlockProfilingTest.class + .getClassLoader() + .getResource("overrides.jfp") + .getFile(); + List command = + new ArrayList<>( + Arrays.asList( + javaPath(), + "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), + "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), + "-javaagent:" + agentShadowJar(), + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Ddd.service.name=smoke-test-objectwait-taskblock", + "-Ddd.env=smoketest", + "-Ddd.version=99", + "-Ddd.profiling.enabled=true", + "-Ddd.profiling.ddprof.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_AUXILIARY_TYPE + "=async", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_ENABLED + "=true", + "-Ddd." + ProfilingConfig.PROFILING_DATADOG_PROFILER_WALL_INTERVAL + "=10ms", + "-Ddd.profiling.agentless=false", + "-Ddd.profiling.start-delay=0", + "-Ddd." + ProfilingConfig.PROFILING_START_FORCE_FIRST + "=true", + "-Ddd.profiling.upload.period=1", + "-Ddd.profiling.hotspots.enabled=true", + "-Ddd." + ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES_SPAN_NAME_ENABLED + "=true", + "-Ddd.profiling.debug.dump_path=" + dumpDir, + "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", + "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug", + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UnlockCommercialFeatures", + "-XX:+FlightRecorder", + "-Ddd." + ProfilingConfig.PROFILING_TEMPLATE_OVERRIDE_FILE + "=" + templateOverride, + "-cp", + System.getProperty("java.class.path"), + ObjectWaitTaskBlockForkedApp.class.getName())); + if (System.getenv("TEST_LIBASYNC") != null) { + command.add( + command.size() - 3, + "-Ddd." + + ProfilingConfig.PROFILING_DATADOG_PROFILER_LIBPATH + + "=" + + System.getenv("TEST_LIBASYNC")); + } + + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.directory(new File(buildDirectory())); + processBuilder.environment().put("JAVA_HOME", System.getProperty("java.home")); + processBuilder.redirectErrorStream(true); + processBuilder.redirectOutput(ProcessBuilder.Redirect.to(logFilePath.toFile())); + return processBuilder; + } + + private JfrStats loadStats() throws Exception { + JfrStats stats = new JfrStats(); + List jfrFiles; + try (java.util.stream.Stream files = Files.walk(dumpDir)) { + jfrFiles = + files + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".jfr")) + .collect(Collectors.toList()); + } + for (Path jfrFile : jfrFiles) { + stats.add(loadEvents(jfrFile)); + } + return stats; + } + + private IItemCollection loadEvents(Path path) { + try { + return JfrLoaderToolkit.loadEvents(extractLastJfrStream(path).toFile()); + } catch (Exception e) { + throw new RuntimeException("Failed to load JFR " + path, e); + } + } + + private Path extractLastJfrStream(Path path) throws IOException { + byte[] data = Files.readAllBytes(path); + int lastMagic = lastIndexOf(data, JFR_MAGIC); + if (lastMagic <= 0) { + return path; + } + + Path extracted = dumpDir.resolve(path.getFileName() + ".ddprof.jfr"); + Files.write(extracted, Arrays.copyOfRange(data, lastMagic, data.length)); + return extracted; + } + + private static int lastIndexOf(byte[] data, byte[] needle) { + for (int i = data.length - needle.length; i >= 0; i--) { + boolean match = true; + for (int j = 0; j < needle.length; j++) { + if (data[i + j] != needle[j]) { + match = false; + break; + } + } + if (match) { + return i; + } + } + return -1; + } + + private boolean logHasObjectWaitInstrumentationError() throws IOException { + String log = new String(Files.readAllBytes(logFilePath), StandardCharsets.UTF_8); + return log.contains("NoClassDefFoundError") + || log.contains("Failed to handle exception in instrumentation for java.lang.Object"); + } + + public static final class ObjectWaitTaskBlockForkedApp { + private static final int WAIT_ITERATIONS = 20; + private static final long LONG_WAIT_MILLIS = 50L; + private static final Object BLOCKER = new Object(); + + public static void main(String[] args) throws Exception { + ObjectWaitTaskBlockForkedApp app = new ObjectWaitTaskBlockForkedApp(GlobalTracer.get()); + app.runActiveSpanWaits(); + app.runSpanlessWaits(); + app.runTooShortWaits(); + Thread.sleep(1500); + } + + private final Tracer tracer; + + private ObjectWaitTaskBlockForkedApp(Tracer tracer) { + this.tracer = tracer; + } + + private void runActiveSpanWaits() throws InterruptedException { + for (int i = 0; i < WAIT_ITERATIONS; i++) { + Span span = tracer.buildSpan("objectwait.active").start(); + try (Scope scope = tracer.activateSpan(span)) { + synchronized (BLOCKER) { + BLOCKER.wait(LONG_WAIT_MILLIS); + } + } finally { + span.finish(); + } + } + } + + private void runSpanlessWaits() throws InterruptedException { + for (int i = 0; i < WAIT_ITERATIONS; i++) { + synchronized (BLOCKER) { + BLOCKER.wait(LONG_WAIT_MILLIS); + } + } + } + + private void runTooShortWaits() throws InterruptedException { + // Exercises the wait(long, int) -> wait(long) routing on JDK 21+. The native task-block + // threshold is 1 ms, so a 1-ms request typically rounds out just over it, but the path + // coverage matters more than the duration here; the assertion focuses on instrumentation + // not crashing rather than which side of the threshold the interval lands on. + for (int i = 0; i < WAIT_ITERATIONS; i++) { + Span span = tracer.buildSpan("objectwait.too-short").start(); + try (Scope scope = tracer.activateSpan(span)) { + synchronized (BLOCKER) { + BLOCKER.wait(0L, 1); + } + } finally { + span.finish(); + } + } + } + } + + private static final class JfrStats { + private long taskBlockCount; + private long taskBlockEmitted; + private long taskBlocksWithNonZeroBlocker; + private boolean hasZeroSpanId; + private boolean hasZeroLocalRootSpanId; + private boolean hasMissingEventThread; + private boolean hasExpectedOperation; + private boolean hasNonZeroUnblockingSpanId; + + private void add(IItemCollection events) { + addTaskBlocks(events); + addWallClockEpochs(events); + } + + private void addTaskBlocks(IItemCollection events) { + IItemCollection taskBlocks = events.apply(ItemFilters.type("datadog.TaskBlock")); + for (IItemIterable items : taskBlocks) { + IMemberAccessor spanIdAccessor = SPAN_ID.getAccessor(items.getType()); + IMemberAccessor localRootSpanIdAccessor = + LOCAL_ROOT_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor blockerAccessor = BLOCKER.getAccessor(items.getType()); + IMemberAccessor unblockingSpanIdAccessor = + UNBLOCKING_SPAN_ID.getAccessor(items.getType()); + IMemberAccessor eventThreadAccessor = + JdkAttributes.EVENT_THREAD_NAME.getAccessor(items.getType()); + IMemberAccessor operationAccessor = OPERATION.getAccessor(items.getType()); + for (IItem item : items) { + String operation = operationAccessor == null ? null : operationAccessor.getMember(item); + // Filter strictly to events emitted by our forked app; the JVM may emit other + // TaskBlock events (LockSupport from agent code, etc.) that we don't want to mix in. + if (!"objectwait.active".equals(operation) && !"objectwait.too-short".equals(operation)) { + continue; + } + taskBlockCount++; + long spanId = spanIdAccessor.getMember(item).longValue(); + long localRootSpanId = localRootSpanIdAccessor.getMember(item).longValue(); + long blocker = blockerAccessor.getMember(item).longValue(); + long unblockingSpanId = unblockingSpanIdAccessor.getMember(item).longValue(); + String eventThread = + eventThreadAccessor == null ? null : eventThreadAccessor.getMember(item); + hasZeroSpanId |= spanId == 0; + hasZeroLocalRootSpanId |= localRootSpanId == 0; + hasMissingEventThread |= eventThread == null || eventThread.isEmpty(); + hasExpectedOperation |= "objectwait.active".equals(operation); + if (blocker != 0) { + taskBlocksWithNonZeroBlocker++; + } + if (unblockingSpanId != 0) { + hasNonZeroUnblockingSpanId = true; + } + } + } + } + + private void addWallClockEpochs(IItemCollection events) { + IItemCollection epochs = events.apply(ItemFilters.type("datadog.WallClockSamplingEpoch")); + for (IItemIterable items : epochs) { + IMemberAccessor emittedAccessor = + TASK_BLOCK_EMITTED.getAccessor(items.getType()); + for (IItem item : items) { + taskBlockEmitted += emittedAccessor.getMember(item).longValue(); + } + } + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java index 4accced983a..3e67cf48fa3 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ProfilingContextIntegration.java @@ -34,6 +34,36 @@ default int encodeResourceName(CharSequence constant) { return 0; } + /** Returns the current TSC tick count for the calling thread. */ + default long getCurrentTicks() { + return 0L; + } + + /** + * Emits a TaskBlock event covering a blocking interval on the current thread. + * + * @param startTicks TSC tick at block entry + * @param spanId the span ID active when blocking began + * @param rootSpanId the local root span ID active when blocking began + * @param blocker identity hash code of the blocking object, or 0 if none + * @param unblockingSpanId the span ID of the thread that unblocked this thread, or 0 if unknown + */ + default void recordTaskBlock( + long startTicks, long spanId, long rootSpanId, long blocker, long unblockingSpanId) {} + + /** + * Called when the current thread is about to enter {@code LockSupport.park*}. Native code can + * suppress wall-clock signals for the park interval and record the start tick for off-CPU + * analysis. + */ + default void parkEnter(long spanId, long rootSpanId) {} + + /** + * Called when the current thread has returned from {@code LockSupport.park*}. Clears the park + * state and may emit a TaskBlock JFR event. + */ + default void parkExit(long blocker, long unblockingSpanId) {} + String name(); final class NoOp implements ProfilingContextIntegration { diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 827230f46ae..913c89e93be 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -7873,6 +7873,14 @@ "aliases": ["DD_TRACE_INTEGRATION_LIBERTY_ENABLED", "DD_INTEGRATION_LIBERTY_ENABLED"] } ], + "DD_TRACE_LOCK_SUPPORT_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_LOCK_SUPPORT_ENABLED", "DD_INTEGRATION_LOCK_SUPPORT_ENABLED"] + } + ], "DD_TRACE_LOG4J_1_ENABLED": [ { "version": "A", @@ -8297,6 +8305,14 @@ "aliases": ["DD_OBFUSCATION_QUERY_STRING_REGEXP"] } ], + "DD_TRACE_OBJECT_WAIT_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": ["DD_TRACE_INTEGRATION_OBJECT_WAIT_ENABLED", "DD_INTEGRATION_OBJECT_WAIT_ENABLED"] + } + ], "DD_TRACE_OGNL_ENABLED": [ { "version": "A", diff --git a/settings.gradle.kts b/settings.gradle.kts index bd5aaceffaa..6716528fb8b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -328,6 +328,8 @@ include( ":dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin", ":dd-java-agent:instrumentation:datadog:profiling:enable-wallclock-profiling", ":dd-java-agent:instrumentation:datadog:profiling:exception-profiling", + ":dd-java-agent:instrumentation:datadog:profiling:lock-support", + ":dd-java-agent:instrumentation:datadog:profiling:object-wait", ":dd-java-agent:instrumentation:datadog:tracing:trace-annotation", ":dd-java-agent:instrumentation:datanucleus-4.0.5", ":dd-java-agent:instrumentation:datastax-cassandra:datastax-cassandra-3.0",