From ccb871592cd2bc05211aea5b95b71f7012146559 Mon Sep 17 00:00:00 2001 From: Asish Kumar Date: Thu, 30 Apr 2026 03:19:31 +0530 Subject: [PATCH 1/2] feat: run Java dedup as java agent Signed-off-by: Asish Kumar --- .github/workflows/java-agent.yml | 33 +++ .woodpecker/build.yml | 3 + README.md | 91 +++---- keploy-sdk/pom.xml | 42 +++ .../io/keploy/dedup/KeployDedupAgent.java | 68 ++++- scripts/smoke-javaagent.sh | 241 ++++++++++++++++++ 6 files changed, 421 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/java-agent.yml create mode 100755 scripts/smoke-javaagent.sh diff --git a/.github/workflows/java-agent.yml b/.github/workflows/java-agent.yml new file mode 100644 index 0000000..4ecb7da --- /dev/null +++ b/.github/workflows/java-agent.yml @@ -0,0 +1,33 @@ +name: Java Agent + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + verify: + name: JDK ${{ matrix.java-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + java-version: ["8", "17", "21"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{ matrix.java-version }} + cache: maven + + - name: Build + run: mvn -B -DskipTests clean verify + + - name: Smoke test Java agent + run: ./scripts/smoke-javaagent.sh diff --git a/.woodpecker/build.yml b/.woodpecker/build.yml index d9b863a..59356d5 100644 --- a/.woodpecker/build.yml +++ b/.woodpecker/build.yml @@ -18,13 +18,16 @@ steps: image: maven:3.9-eclipse-temurin-8 commands: - mvn -B -DskipTests clean verify + - ./scripts/smoke-javaagent.sh build-jdk-17: image: maven:3.9-eclipse-temurin-17 commands: - mvn -B -DskipTests clean verify + - ./scripts/smoke-javaagent.sh build-jdk-21: image: maven:3.9-eclipse-temurin-21 commands: - mvn -B -DskipTests clean verify + - ./scripts/smoke-javaagent.sh diff --git a/README.md b/README.md index 70795a7..0832e3f 100644 --- a/README.md +++ b/README.md @@ -23,55 +23,41 @@ Coverage is collected at per-testcase granularity, not process granularity. ## How to Use -### 1. Add the SDK +### 1. Download the Keploy Java Agent -Add `keploy-sdk` to your application: +Download the `keploy-sdk` jar and keep it outside your application dependencies. The jar is a Java agent and should be attached only when you run Keploy dynamic deduplication. ```xml - - io.keploy - keploy-sdk - 2.0.0 - + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.1 + + + copy-keploy-java-agent + package + + copy + + + + + io.keploy + keploy-sdk + 2.0.1 + ${project.build.directory} + keploy-sdk.jar + + + + + + ``` -### 2. Activate the Agent +The SDK no longer has to be added to `dependencies`, and application code should not import `io.keploy.*` classes for dynamic deduplication. -For Spring Boot, import the middleware in your application: - -```java -import io.keploy.servlet.KeployMiddleware; -import org.springframework.context.annotation.Import; - -@Import(KeployMiddleware.class) -public class Application { -} -``` - -For servlet-based applications, register the filter early in `web.xml`: - -```xml - - middleware - io.keploy.servlet.KeployMiddleware - - - middleware - /* - -``` - -The middleware starts the Java dedup control server automatically. - -For Jakarta Servlet stacks, non-servlet frameworks, or any application where the `javax.servlet` filter is not available, start the agent directly during application startup: - -```java -import io.keploy.dedup.KeployDedupAgent; - -KeployDedupAgent.start(); -``` - -### 3. Run the App with the JaCoCo Java Agent +### 2. Run the App with the Keploy and JaCoCo Java Agents The dedup agent reads coverage in-process via JaCoCo's runtime API (`org.jacoco.agent.rt.RT.getAgent()`), so attaching the JaCoCo Java agent is the only runtime requirement in the common cases below: @@ -79,22 +65,27 @@ The dedup agent reads coverage in-process via JaCoCo's runtime API (`org.jacoco. - packaged `java -jar` runs where the application classes live inside the executable jar ```bash -java -javaagent:/path/to/jacocoagent.jar -jar your-app.jar +java \ + -javaagent:/path/to/keploy-sdk.jar \ + -javaagent:/path/to/jacocoagent.jar \ + -jar your-app.jar ``` If the in-process API is unavailable (for example because the JaCoCo agent is loaded into an isolated classloader), the SDK transparently falls back to JaCoCo's TCP server mode. To use the fallback explicitly, start JaCoCo in `tcpserver` mode and set `KEPLOY_JACOCO_HOST` / `KEPLOY_JACOCO_PORT`: ```bash -java -javaagent:/path/to/jacocoagent.jar=address=127.0.0.1,port=36320,output=tcpserver \ +java \ + -javaagent:/path/to/keploy-sdk.jar \ + -javaagent:/path/to/jacocoagent.jar=address=127.0.0.1,port=36320,output=tcpserver \ -jar your-app.jar ``` -### 4. Replay with Keploy Enterprise +### 3. Replay with Keploy Enterprise Run replay with dynamic dedup enabled: ```bash -keploy test -c "java -javaagent:/path/to/jacocoagent.jar -jar your-app.jar" \ +keploy test -c "java -javaagent:/path/to/keploy-sdk.jar -javaagent:/path/to/jacocoagent.jar -jar your-app.jar" \ --dedup \ --language java ``` @@ -128,8 +119,8 @@ Without a shared `/tmp`, dedup will not work inside containers because Enterpris - `KEPLOY_JACOCO_HOST`: JaCoCo TCP host used when the in-process runtime API is unavailable. Default: `127.0.0.1` - `KEPLOY_JACOCO_PORT`: JaCoCo TCP port used when the in-process runtime API is unavailable. Default: `36320` -- `KEPLOY_JAVA_CLASS_DIRS`: optional comma-separated class or jar locations to analyze for executed lines when your build output lives outside the standard locations -- `KEPLOY_JAVA_CLASSPATH_FALLBACK`: scans the full classpath if standard class roots and the executable jar do not provide application classes. Default: `false` +- `KEPLOY_JAVA_CLASS_DIRS`: optional comma-separated class, jar, war, ear, or zip locations to analyze for executed lines when your build output lives outside the standard locations +- `KEPLOY_JAVA_CLASSPATH_FALLBACK`: scans the full classpath if standard class roots and the executable archive do not provide application classes. Default: `true` - `KEPLOY_JAVA_DEDUP_DISABLED`: disables the Java dedup agent when set to `true`, `1`, or `yes` ## Sample @@ -138,4 +129,4 @@ For a working reference, see the Java dedup sample in `keploy/samples-java`: - `samples-java/java-dedup` -That sample is used in CI to validate Java dynamic dedup for JDK 8, 17, and 21 across native, Docker, and restricted Docker runs. +That sample is used in CI to validate Java dynamic dedup for JDK 8, 17, and 21 across native, classpath, Docker, distroless, and restricted Docker runs. diff --git a/keploy-sdk/pom.xml b/keploy-sdk/pom.xml index 2bf87c0..64f16f9 100644 --- a/keploy-sdk/pom.xml +++ b/keploy-sdk/pom.xml @@ -78,6 +78,48 @@ + + org.apache.maven.plugins + maven-shade-plugin + 3.5.3 + + + shade-java-agent + package + + shade + + + false + + + + io.keploy.dedup.KeployDedupAgent + io.keploy.dedup.KeployDedupAgent + false + false + ${project.name} + ${project.version} + io.keploy.sdk + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + module-info.class + META-INF/versions/*/module-info.class + + + + + + + org.apache.maven.plugins maven-javadoc-plugin diff --git a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java index 2465640..30cd22f 100644 --- a/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java +++ b/keploy-sdk/src/main/java/io/keploy/dedup/KeployDedupAgent.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.lang.instrument.Instrumentation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetAddress; @@ -39,6 +40,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -65,11 +67,32 @@ public final class KeployDedupAgent { private static final int SOCKET_BACKLOG = 50; private static final AtomicBoolean STARTED = new AtomicBoolean(false); + private static final AtomicBoolean SHUTDOWN_HOOK_REGISTERED = new AtomicBoolean(false); private static volatile CommandServer commandServer; private KeployDedupAgent() { } + /** + * JVM entrypoint used when the SDK is attached with {@code -javaagent}. + * + * @param agentArgs optional Java agent arguments + * @param instrumentation JVM instrumentation handle + */ + public static void premain(String agentArgs, Instrumentation instrumentation) { + start(); + } + + /** + * JVM entrypoint used when the SDK is attached to an already running JVM. + * + * @param agentArgs optional Java agent arguments + * @param instrumentation JVM instrumentation handle + */ + public static void agentmain(String agentArgs, Instrumentation instrumentation) { + start(); + } + /** * Starts the background control socket listener used by Keploy replay. * @@ -91,6 +114,7 @@ public static boolean start() { thread.setDaemon(true); commandServer = server; thread.start(); + registerShutdownHook(); return true; } @@ -120,6 +144,22 @@ private static boolean isDisabled() { || isTruthy(System.getProperty("keploy.java.dedup.disabled")); } + private static void registerShutdownHook() { + if (!SHUTDOWN_HOOK_REGISTERED.compareAndSet(false, true)) { + return; + } + try { + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + KeployDedupAgent.stop(); + } + }, "keploy-java-dedup-shutdown")); + } catch (IllegalStateException ignored) { + // The VM is already shutting down; the process exit will reclaim resources. + } + } + private static boolean diagnosticsEnabled() { return isTruthy(System.getenv("KEPLOY_JAVA_DEDUP_DIAGNOSTICS")) || isTruthy(System.getProperty("keploy.java.dedup.diagnostics")); @@ -802,13 +842,13 @@ private List applicationRoots() { private boolean isClasspathFallbackEnabled() { return isTruthy(envOrProperty("KEPLOY_JAVA_CLASSPATH_FALLBACK", - "keploy.java.classpath.fallback", "false")); + "keploy.java.classpath.fallback", "true")); } private List executableArchiveRoots() { LinkedHashSet roots = new LinkedHashSet<>(); - addJarRoot(roots, firstCommandToken(System.getProperty("sun.java.command", ""))); + addArchiveRoot(roots, firstCommandToken(System.getProperty("sun.java.command", ""))); String classpath = System.getProperty("java.class.path", ""); if (classpath.trim().isEmpty()) { @@ -817,12 +857,12 @@ private List executableArchiveRoots() { String[] parts = classpath.split(Pattern.quote(File.pathSeparator)); if (parts.length == 1) { - addJarRoot(roots, parts[0]); + addArchiveRoot(roots, parts[0]); } return new ArrayList<>(roots); } - private void addJarRoot(Set roots, String rawPath) { + private void addArchiveRoot(Set roots, String rawPath) { if (rawPath == null) { return; } @@ -836,11 +876,19 @@ private void addJarRoot(Set roots, String rawPath) { if (!file.isAbsolute()) { file = new File(System.getProperty("user.dir"), path); } - if (file.isFile() && file.getName().endsWith(".jar")) { + if (file.isFile() && isArchive(file)) { roots.add(file); } } + private boolean isArchive(File file) { + String name = file.getName().toLowerCase(Locale.ROOT); + return name.endsWith(".jar") + || name.endsWith(".war") + || name.endsWith(".ear") + || name.endsWith(".zip"); + } + private String firstCommandToken(String command) { if (command == null) { return ""; @@ -879,7 +927,7 @@ private List classpathRoots() { for (String part : parts) { if (!part.trim().isEmpty()) { File file = new File(part.trim()); - if (file.isDirectory() || file.getName().endsWith(".jar")) { + if (file.isDirectory() || isArchive(file)) { roots.add(file); } } @@ -895,7 +943,7 @@ private void scanRoots(List roots, Map output) { } if (root.isDirectory()) { scanDirectory(root, output); - } else if (root.isFile() && root.getName().endsWith(".jar")) { + } else if (root.isFile() && isArchive(root)) { scanJar(root, output); } } @@ -972,6 +1020,12 @@ private String classKeyFromJarEntry(String entryName) { private boolean shouldSkipClass(String name) { return name.endsWith("module-info.class") || name.endsWith("package-info.class") + || name.startsWith("io/keploy/dedup/") + || name.startsWith("io/keploy/servlet/") + || name.startsWith("org/jacoco/") + || name.startsWith("org/objectweb/asm/") + || name.startsWith("org/newsclub/net/unix/") + || name.startsWith("com/google/gson/") || name.contains("$Mockito") || name.contains("Test.class"); } diff --git a/scripts/smoke-javaagent.sh b/scripts/smoke-javaagent.sh new file mode 100755 index 0000000..39058ee --- /dev/null +++ b/scripts/smoke-javaagent.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +module_dir="${repo_root}/keploy-sdk" +mkdir -p "${module_dir}/target" +agent_jar="$(find "${module_dir}/target" -maxdepth 1 -type f -name 'keploy-sdk-*.jar' \ + ! -name 'original-*' ! -name '*-sources.jar' ! -name '*-javadoc.jar' | sort | head -n 1)" + +if [[ -z "${agent_jar}" || ! -f "${agent_jar}" ]]; then + echo "Missing agent jar under ${module_dir}/target" >&2 + echo "Run mvn -B -DskipTests package first." >&2 + exit 1 +fi + +mvn -q -f "${repo_root}/pom.xml" org.apache.maven.plugins:maven-dependency-plugin:3.6.1:copy \ + -Dartifact=org.jacoco:org.jacoco.agent:0.8.12:jar:runtime \ + -DoutputDirectory="${module_dir}/target" \ + -DdestFileName=jacocoagent.jar + +jacoco_jar="$(find "${module_dir}/target" -maxdepth 1 -type f \( \ + -name 'jacocoagent.jar' -o -name 'org.jacoco.agent-*-runtime.jar' \) | sort | head -n 1)" +if [[ -z "${jacoco_jar}" || ! -f "${jacoco_jar}" ]]; then + echo "Missing JaCoCo runtime agent jar under ${module_dir}/target" >&2 + exit 1 +fi + +work_dir="$(mktemp -d)" +cleanup() { + rm -rf "${work_dir}" + rm -f /tmp/coverage_control.sock /tmp/coverage_data.sock /tmp/keploy-sdk-smoke-*.exec +} +trap cleanup EXIT + +mkdir -p "${work_dir}/src/smoke" "${work_dir}/classes" + +cat > "${work_dir}/src/smoke/Work.java" <<'JAVA' +package smoke; + +final class Work { + private Work() { + } + + static int exercise(int input) { + int value = input + 1; + if (value > 1) { + value = value * 2; + } + return value; + } +} +JAVA + +cat > "${work_dir}/src/smoke/SmokeHarness.java" <<'JAVA' +package smoke; + +import org.newsclub.net.unix.AFUNIXServerSocket; +import org.newsclub.net.unix.AFUNIXSocket; +import org.newsclub.net.unix.AFUNIXSocketAddress; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +public final class SmokeHarness { + private static final File CONTROL_SOCKET = new File("/tmp/coverage_control.sock"); + private static final File DATA_SOCKET = new File("/tmp/coverage_data.sock"); + + public static void main(String[] args) throws Exception { + String mode = args.length == 0 ? "unknown" : args[0]; + delete(DATA_SOCKET); + + AtomicReference payload = new AtomicReference(); + CountDownLatch received = new CountDownLatch(1); + Thread receiver = new Thread(new Runnable() { + @Override + public void run() { + try (AFUNIXServerSocket server = AFUNIXServerSocket.newInstance()) { + server.bind(AFUNIXSocketAddress.of(DATA_SOCKET), 1); + try (AFUNIXSocket socket = server.accept()) { + payload.set(readAll(socket.getInputStream())); + received.countDown(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + }, "coverage-data-receiver"); + receiver.setDaemon(true); + receiver.start(); + + waitFor(CONTROL_SOCKET); + waitFor(DATA_SOCKET); + + command("START test-set-0/" + mode); + if (Work.exercise(1) != 4) { + throw new IllegalStateException("unexpected application result"); + } + command("END test-set-0/" + mode); + + if (!received.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("timed out waiting for coverage payload in " + mode); + } + + String json = payload.get(); + if (json == null + || !json.contains("\"id\":\"test-set-0/" + mode + "\"") + || !json.contains("Work.java")) { + throw new IllegalStateException("unexpected coverage payload for " + mode + ": " + json); + } + System.out.println(mode + ": " + json); + } + + private static void command(String command) throws Exception { + try (AFUNIXSocket socket = AFUNIXSocket.newInstance()) { + socket.connect(AFUNIXSocketAddress.of(CONTROL_SOCKET), 3000); + OutputStream output = socket.getOutputStream(); + output.write((command + "\n").getBytes(StandardCharsets.UTF_8)); + output.flush(); + String ack = readAll(socket.getInputStream()); + if (!ack.contains("ACK")) { + throw new IllegalStateException("missing ACK for " + command + ": " + ack); + } + } + } + + private static void waitFor(File file) throws InterruptedException { + long deadline = System.currentTimeMillis() + 10000L; + while (System.currentTimeMillis() < deadline) { + if (file.exists()) { + return; + } + Thread.sleep(50L); + } + throw new IllegalStateException("timed out waiting for " + file); + } + + private static String readAll(InputStream input) throws Exception { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int read; + while ((read = input.read(buffer)) != -1) { + output.write(buffer, 0, read); + } + return new String(output.toByteArray(), StandardCharsets.UTF_8); + } + + private static void delete(File file) { + if (file.exists() && !file.delete()) { + throw new IllegalStateException("failed to delete " + file); + } + } +} +JAVA + +javac -cp "${agent_jar}" -d "${work_dir}/classes" \ + "${work_dir}/src/smoke/Work.java" \ + "${work_dir}/src/smoke/SmokeHarness.java" + +run_smoke() { + local mode="${1:?mode required}" + shift + + echo "Running Java agent smoke: ${mode}" + rm -f /tmp/coverage_control.sock /tmp/coverage_data.sock "/tmp/keploy-sdk-smoke-${mode}.exec" + "$@" +} + +java_with_agents() { + local mode="${1:?mode required}" + shift + + java \ + -javaagent:"${agent_jar}" \ + -javaagent:"${jacoco_jar}=destfile=/tmp/keploy-sdk-smoke-${mode}.exec" \ + "$@" +} + +mkdir -p "${work_dir}/maven/target/classes" "${work_dir}/gradle/build/classes/java/main" +cp -R "${work_dir}/classes/." "${work_dir}/maven/target/classes/" +cp -R "${work_dir}/classes/." "${work_dir}/gradle/build/classes/java/main/" + +( + cd "${work_dir}/maven" + run_smoke "maven-classes" \ + java_with_agents "maven-classes" \ + -cp "target/classes:${agent_jar}" \ + smoke.SmokeHarness "maven-classes" +) + +( + cd "${work_dir}/gradle" + run_smoke "gradle-classes" \ + java_with_agents "gradle-classes" \ + -cp "build/classes/java/main:${agent_jar}" \ + smoke.SmokeHarness "gradle-classes" +) + +run_smoke "classpath-fallback" \ + java_with_agents "classpath-fallback" \ + -cp "${work_dir}/classes:${agent_jar}" \ + smoke.SmokeHarness "classpath-fallback" + +mkdir -p "${work_dir}/jar/lib" +cp "${agent_jar}" "${work_dir}/jar/lib/keploy-sdk.jar" +cat > "${work_dir}/jar/MANIFEST.MF" < Date: Thu, 30 Apr 2026 15:21:52 +0530 Subject: [PATCH 2/2] chore: prepare java agent 2.0.0 Signed-off-by: Asish Kumar --- README.md | 2 +- keploy-sdk/pom.xml | 4 ++-- pom.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0832e3f..29f816f 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Download the `keploy-sdk` jar and keep it outside your application dependencies. io.keploy keploy-sdk - 2.0.1 + 2.0.0 ${project.build.directory} keploy-sdk.jar diff --git a/keploy-sdk/pom.xml b/keploy-sdk/pom.xml index 64f16f9..6e154bc 100644 --- a/keploy-sdk/pom.xml +++ b/keploy-sdk/pom.xml @@ -7,11 +7,11 @@ java-sdk io.keploy - 1.0.0-SNAPSHOT + 2.0.0 keploy-sdk - 0.0.1-SNAPSHOT + 2.0.0 Keploy Java Coverage Agent Java dynamic dedup coverage agent for Keploy Enterprise https://github.com/keploy/java-sdk diff --git a/pom.xml b/pom.xml index c84a8d8..23f94fc 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ io.keploy java-sdk - 1.0.0-SNAPSHOT + 2.0.0 pom Keploy Java Coverage Agent