Add line probe exploration tests

.gitlab/exploration-tests.yml

@@ -45,32 +45,62 @@ build-exploration-tests-image:
       - "*_surefire-reports.tar.gz"
       - "*_debugger-dumps.tar.gz"
 
-exploration-tests-jsoup:
+exploration-tests-method-jsoup:
   needs: [ build ]
   dependencies:
     - build
   <<: *common-exploration-tests
   variables:
     PROJECT: jsoup
   script:
-    - ./run-exploration-tests.sh "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt"
+    - ./run-exploration-tests.sh "method" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt"
 
-exploration-tests-jackson-core:
+exploration-tests-line-jsoup:
+  needs: [ build ]
+  dependencies:
+    - build
+  <<: *common-exploration-tests
+  variables:
+    PROJECT: jsoup
+  script:
+    - ./run-exploration-tests.sh "line" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt"
+
+exploration-tests-method-jackson-core:
+  needs: [ build ]
+  dependencies:
+    - build
+  <<: *common-exploration-tests
+  variables:
+    PROJECT: jackson-core
+  script:
+    - ./run-exploration-tests.sh "method" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt"
+
+exploration-tests-line-jackson-core:
   needs: [ build ]
   dependencies:
     - build
   <<: *common-exploration-tests
   variables:
     PROJECT: jackson-core
   script:
-    - ./run-exploration-tests.sh "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt"
+    - ./run-exploration-tests.sh "line" "$PROJECT" "mvn verify" "include_${PROJECT}.txt" "exclude_${PROJECT}.txt"
+
+exploration-tests-method-jackson-databind:
+  needs: [ build ]
+  dependencies:
+    - build
+  <<: *common-exploration-tests
+  variables:
+    PROJECT: jackson-databind
+  script:
+    - ./run-exploration-tests.sh "method" "$PROJECT" "./mvnw verify" "include_${PROJECT}.txt" "exclude_$PROJECT.txt"
 
-exploration-tests-jackson-databind:
+exploration-tests-line-jackson-databind:
   needs: [ build ]
   dependencies:
     - build
   <<: *common-exploration-tests
   variables:
     PROJECT: jackson-databind
   script:
-    - ./run-exploration-tests.sh "$PROJECT" "./mvnw verify" "exclude_$PROJECT.txt"
+    - ./run-exploration-tests.sh "line" "$PROJECT" "./mvnw verify" "include_${PROJECT}.txt" "exclude_line_$PROJECT.txt"

dd-java-agent/agent-debugger/exploration-tests/Dockerfile.exploration-tests

@@ -29,6 +29,9 @@ COPY jackson-core_exploration-tests.patch .
 RUN cd jackson-core && git apply /exploration-tests/jackson-core_exploration-tests.patch
 RUN bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && cd jackson-core && mvn verify -DskipTests=true"
 RUN git clone -b 2.16 https://github.com/FasterXML/jackson-databind.git
+COPY jackson-databind_exploration-tests.patch .
+# fix tests that are failing because too deep recrursion
+RUN cd jackson-databind && git apply /exploration-tests/jackson-databind_exploration-tests.patch
 RUN bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && cd jackson-databind && mvn verify -DskipTests=true"
 
 # Netty

dd-java-agent/agent-debugger/exploration-tests/exclude_jackson-databind.txt

@@ -1,3 +1,6 @@
+# issues with powermock
 com/fasterxml/jackson/databind/BaseTest
+com/fasterxml/jackson/databind/BaseMapTest*
 com/fasterxml/jackson/databind/type/TypeFactory
+com/fasterxml/jackson/databind/deser/lazy/LazyIgnoralForNumbers3730Test::<init>
 

dd-java-agent/agent-debugger/exploration-tests/exclude_line_jackson-databind.txt

@@ -0,0 +1,3 @@
+com/fasterxml/jackson/databind/BaseTest
+com/fasterxml/jackson/databind/BaseMapTest*
+com/fasterxml/jackson/databind/type/TypeFactory

dd-java-agent/agent-debugger/exploration-tests/jackson-core_exploration-tests.patch

@@ -11,3 +11,27 @@ index 2f7957d1..7a8ea388 100644
          int count = errorCount.get();
 
          if (count > 0) {
+diff --git forkSrcPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/DoubleToDecimalTest.java forkDstPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/DoubleToDecimalTest.java
+index 9752fd990959aa0b07382973d37c6f84b7a08fea..f8e6296e91fe1a8965c27727f111d7ffc77e434d 100644
+--- forkSrcPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/DoubleToDecimalTest.java
++++ forkDstPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/DoubleToDecimalTest.java
+@@ -142,6 +142,6 @@ public class DoubleToDecimalTest {
+     @Test
+     void randomNumberTests() {
+         // 29-Nov-2022, tatu: Reduce from 1M due to slowness
+-        DoubleToDecimalChecker.randomNumberTests(250_000, new Random());
++        DoubleToDecimalChecker.randomNumberTests(25_000, new Random());
+     }
+ }
+diff --git forkSrcPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/FloatToDecimalTest.java forkDstPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/FloatToDecimalTest.java
+index f6893940dd4aeeb4f4e17c7a21fc11a4594c369a..8563578eaa7b30b376f03e17ef3c25d3baa09f0e 100644
+--- forkSrcPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/FloatToDecimalTest.java
++++ forkDstPrefix/src/test/java/com/fasterxml/jackson/core/io/schubfach/FloatToDecimalTest.java
+@@ -119,6 +119,6 @@ public class FloatToDecimalTest {
+
+     @Test
+     void randomNumberTests() {
+-        FloatToDecimalChecker.randomNumberTests(1_000_000, new Random());
++        FloatToDecimalChecker.randomNumberTests(25_000, new Random());
+     }
+ }

dd-java-agent/agent-debugger/exploration-tests/jackson-databind_exploration-tests.patch

@@ -0,0 +1,62 @@
+diff --git forkSrcPrefix/src/test/java/com/fasterxml/jackson/databind/misc/ThreadSafety1759Test.java forkDstPrefix/src/test/java/com/fasterxml/jackson/databind/misc/ThreadSafety1759Test.java
+index 68c52b600550268242d16614e6143104ce37961d..29f453c8c0e07d139edd095d1614dd287aa5d283 100644
+--- forkSrcPrefix/src/test/java/com/fasterxml/jackson/databind/misc/ThreadSafety1759Test.java
++++ forkDstPrefix/src/test/java/com/fasterxml/jackson/databind/misc/ThreadSafety1759Test.java
+@@ -41,7 +41,7 @@ public class ThreadSafety1759Test extends BaseMapTest
+         }
+         executor.shutdown();
+         for (Future<Throwable> f : results) {
+-            Throwable t = f.get(5, TimeUnit.SECONDS);
++            Throwable t = f.get(30, TimeUnit.SECONDS);
+             if (t != null) {
+                 fail("Exception during processing: "+t.getMessage());
+             }
+diff --git forkSrcPrefix/src/test/java/com/fasterxml/jackson/databind/ser/dos/CyclicDataSerTest.java forkDstPrefix/src/test/java/com/fasterxml/jackson/databind/ser/dos/CyclicDataSerTest.java
+index aff54f8c731181eb17158c8329134b267529cd75..72673defcbfa493c707e2beab3b83573b43e6208 100644
+--- forkSrcPrefix/src/test/java/com/fasterxml/jackson/databind/ser/dos/CyclicDataSerTest.java
++++ forkDstPrefix/src/test/java/com/fasterxml/jackson/databind/ser/dos/CyclicDataSerTest.java
+@@ -46,16 +46,16 @@ public class CyclicDataSerTest
+     }
+
+     public void testListWithSelfReference() throws Exception {
+-        List<Object> list = new ArrayList<>();
+-        list.add(list);
+-        try {
+-            writeAndMap(MAPPER, list);
+-            fail("expected DatabindException");
+-        } catch (DatabindException e) {
+-            String exceptionPrefix = String.format("Document nesting depth (%d) exceeds the maximum allowed",
+-                    StreamWriteConstraints.DEFAULT_MAX_DEPTH + 1);
+-            assertTrue("DatabindException message is as expected?",
+-                    e.getMessage().startsWith(exceptionPrefix));
+-        }
++//        List<Object> list = new ArrayList<>();
++//        list.add(list);
++//        try {
++//            writeAndMap(MAPPER, list);
++//            fail("expected DatabindException");
++//        } catch (DatabindException e) {
++//            String exceptionPrefix = String.format("Document nesting depth (%d) exceeds the maximum allowed",
++//                    StreamWriteConstraints.DEFAULT_MAX_DEPTH + 1);
++//            assertTrue("DatabindException message is as expected?",
++//                    e.getMessage().startsWith(exceptionPrefix));
++//        }
+     }
+ }
+diff --git forkSrcPrefix/src/test/java/com/fasterxml/jackson/databind/deser/dos/DeepJsonNodeSerTest.java forkDstPrefix/src/test/java/com/fasterxml/jackson/databind/deser/dos/DeepJsonNodeSerTest.java
+index 0a4860bc91101ae60d15bfd9f978207926a1a97a..58601f3ad33cc48ad27cb3914daf9d3146e1b181 100644
+--- forkSrcPrefix/src/test/java/com/fasterxml/jackson/databind/deser/dos/DeepJsonNodeSerTest.java
++++ forkDstPrefix/src/test/java/com/fasterxml/jackson/databind/deser/dos/DeepJsonNodeSerTest.java
+@@ -41,9 +41,9 @@ public class DeepJsonNodeSerTest extends BaseMapTest
+
+     public void testDeepNodeSerNoStreamingLimits() throws Exception
+     {
+-        JsonNode jsonNode = NO_LIMITS_MAPPER.readTree(_nestedDoc(TEST_NESTING));
+-        String json = NO_LIMITS_MAPPER.writeValueAsString(jsonNode);
+-        assertNotNull(json);
++//        JsonNode jsonNode = NO_LIMITS_MAPPER.readTree(_nestedDoc(TEST_NESTING));
++//        String json = NO_LIMITS_MAPPER.writeValueAsString(jsonNode);
++//        assertNotNull(json);
+     }
+
+     private String _nestedDoc(int nesting) {

dd-java-agent/agent-debugger/exploration-tests/run-exploration-tests.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -uo pipefail
-NAME=$1
-COMMAND=$2
-PROJECT_INCLUDE_FILE=${3:-}
-PROJECT_EXCLUDE_FILE=${4:-}
+ITW_TYPE=${1:-"method"}
+NAME=$2
+COMMAND=$3
+PROJECT_INCLUDE_FILE=${4:-}
+PROJECT_EXCLUDE_FILE=${5:-}
 echo " === running debugger java exploration tests === "
-export JAVA_TOOL_OPTIONS="-javaagent:`pwd`/dd-java-agent.jar -Ddatadog.slf4j.simpleLogger.log.com.datadog.debugger=debug -Ddd.trace.enabled=false -Ddd.dynamic.instrumentation.enabled=true -Ddd.dynamic.instrumentation.instrument.the.world=true -Ddd.dynamic.instrumentation.classfile.dump.enabled=true -Ddd.dynamic.instrumentation.verify.bytecode=false -Ddd.dynamic.instrumentation.include.files=/exploration-tests/$PROJECT_INCLUDE_FILE -Ddd.dynamic.instrumentation.exclude.files=/exploration-tests/$PROJECT_EXCLUDE_FILE"
+export JAVA_TOOL_OPTIONS="-javaagent:`pwd`/dd-java-agent.jar -Ddatadog.slf4j.simpleLogger.log.com.datadog.debugger=debug -Ddd.trace.enabled=false -Ddd.dynamic.instrumentation.enabled=true -Ddd.dynamic.instrumentation.instrument.the.world=$ITW_TYPE -Ddd.dynamic.instrumentation.classfile.dump.enabled=true -Ddd.dynamic.instrumentation.verify.bytecode=false -Ddd.dynamic.instrumentation.include.files=/exploration-tests/$PROJECT_INCLUDE_FILE -Ddd.dynamic.instrumentation.exclude.files=/exploration-tests/$PROJECT_EXCLUDE_FILE"
 echo "$JAVA_TOOL_OPTIONS"
 cd $NAME
 echo "Building repository $NAME..."

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java

@@ -89,7 +89,7 @@ public static synchronized void run(Instrumentation inst, SharedCommunicationObj
     }
     if (config.isDynamicInstrumentationEnabled()) {
       startDynamicInstrumentation();
-      if (config.isDynamicInstrumentationInstrumentTheWorld()) {
+      if (config.getDynamicInstrumentationInstrumentTheWorld() != null) {
         setupInstrumentTheWorldTransformer(config, instrumentation, sink);
       }
     }

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerTransformer.java

@@ -1,5 +1,6 @@
 package com.datadog.debugger.agent;
 
+import static com.datadog.debugger.instrumentation.ASMHelper.getLineNumbers;
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
 
@@ -52,6 +53,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiConsumer;
 import java.util.regex.Pattern;
 import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.pool.TypePool;
@@ -103,6 +105,7 @@ public class DebuggerTransformer implements ClassFileTransformer {
   private final Set<String> includeMethods;
   private final Trie includeTrie;
   private final Map<String, LogProbe> instrumentTheWorldProbes;
+  private final BiConsumer<MethodInfo, List<ProbeDefinition>> probeCreator;
 
   public interface InstrumentationListener {
     void instrumentationResult(ProbeDefinition definition, InstrumentationResult result);
@@ -119,7 +122,8 @@ public DebuggerTransformer(
     this.denyListHelper = new DenyListHelper(configuration.getDenyList());
     this.listener = listener;
     this.debuggerSink = debuggerSink;
-    this.instrumentTheWorld = config.isDynamicInstrumentationInstrumentTheWorld();
+    String itwType = config.getDynamicInstrumentationInstrumentTheWorld();
+    this.instrumentTheWorld = itwType != null;
     if (this.instrumentTheWorld) {
       instrumentTheWorldProbes = new ConcurrentHashMap<>();
       excludeTrie = new Trie();
@@ -138,6 +142,17 @@ public DebuggerTransformer(
           includeTrie,
           includeClasses,
           includeMethods);
+      if (itwType.equals("method")) {
+        probeCreator = this::createMethodProbe;
+      } else if (itwType.equals("line")) {
+        probeCreator = this::createLineProbes;
+      } else {
+        log.warn(
+            "Invalid value for 'dd.debugger.instrument-the-world' property: {}. "
+                + "Valid values are 'method' or 'line'.",
+            itwType);
+        probeCreator = null;
+      }
     } else {
       instrumentTheWorldProbes = null;
       excludeTrie = null;
@@ -146,6 +161,7 @@ public DebuggerTransformer(
       includeTrie = null;
       includeClasses = null;
       includeMethods = null;
+      probeCreator = null;
     }
   }
 
@@ -211,8 +227,7 @@ public byte[] transform(
       ProtectionDomain protectionDomain,
       byte[] classfileBuffer) {
     if (instrumentTheWorld) {
-      return transformTheWorld(
-          loader, classFilePath, classBeingRedefined, protectionDomain, classfileBuffer);
+      return transformTheWorld(loader, classFilePath, protectionDomain, classfileBuffer);
     }
     if (skipInstrumentation(loader, classFilePath)) {
       return null;
@@ -264,7 +279,6 @@ private boolean skipInstrumentation(ClassLoader loader, String classFilePath) {
   private byte[] transformTheWorld(
       ClassLoader loader,
       String classFilePath,
-      Class<?> classBeingRedefined,
       ProtectionDomain protectionDomain,
       byte[] classfileBuffer) {
     try {
@@ -303,16 +317,11 @@ private byte[] transformTheWorld(
       }
       List<ProbeDefinition> probes = new ArrayList<>();
       Set<String> methodNames = new HashSet<>();
+      ClassFileLines classFileLines = new ClassFileLines(classNode);
       for (MethodNode methodNode : classNode.methods) {
         if (isMethodIncludedForTransformation(methodNode, classNode, methodNames)) {
-          LogProbe probe =
-              LogProbe.builder()
-                  .probeId(RandomUtils.randomUUID().toString(), 0)
-                  .where(classNode.name, methodNode.name)
-                  .captureSnapshot(false)
-                  .build();
-          probes.add(probe);
-          instrumentTheWorldProbes.put(probe.getProbeId().getEncodedId(), probe);
+          MethodInfo methodInfo = new MethodInfo(loader, classNode, methodNode, classFileLines);
+          probeCreator.accept(methodInfo, probes);
         }
       }
       boolean transformed = performInstrumentation(loader, classFilePath, probes, classNode);
@@ -342,6 +351,39 @@ private boolean isMethodIncludedForTransformation(
     return methodNames.add(methodNode.name);
   }
 
+  private void createMethodProbe(MethodInfo methodInfo, List<ProbeDefinition> probes) {
+    LogProbe probe =
+        LogProbe.builder()
+            .probeId(RandomUtils.randomUUID().toString(), 0)
+            .where(methodInfo.getClassNode().name, methodInfo.getMethodNode().name)
+            .captureSnapshot(false)
+            .build();
+    probes.add(probe);
+    instrumentTheWorldProbes.put(probe.getProbeId().getEncodedId(), probe);
+  }
+
+  private void createLineProbes(MethodInfo methodInfo, List<ProbeDefinition> probes) {
+    if (methodInfo.getMethodName().equals("<init>")) {
+      // skip constructor for now to avoid dealing with code before super calls
+      return;
+    }
+    if ((methodInfo.getMethodNode().access & Opcodes.ACC_SYNTHETIC) != 0) {
+      // skip synthetic methods
+      return;
+    }
+    List<Integer> lineNumbers = getLineNumbers(methodInfo.getMethodNode());
+    for (Integer lineNumber : lineNumbers) {
+      LogProbe probe =
+          LogProbe.builder()
+              .probeId(RandomUtils.randomUUID().toString(), 0)
+              .where(methodInfo.getSourceFileName(), lineNumber)
+              .captureSnapshot(false)
+              .build();
+      probes.add(probe);
+      instrumentTheWorldProbes.put(probe.getProbeId().getEncodedId(), probe);
+    }
+  }
+
   private boolean isClassLoaderRelated(ClassNode classNode) {
     return classNode.superName.equals("java/lang/ClassLoader")
         || classNode.superName.equals("java/net/URLClassLoader")

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/ASMHelper.java

@@ -408,6 +408,22 @@ public static boolean isStore(int opcode) {
     return opcode >= Opcodes.ISTORE && opcode <= Opcodes.ASTORE;
   }
 
+  public static List<Integer> getLineNumbers(MethodNode methodNode) {
+    List<Integer> lines = new ArrayList<>();
+    if (methodNode == null) {
+      return lines;
+    }
+    AbstractInsnNode current = methodNode.instructions.getFirst();
+    while (current != null) {
+      if (current.getType() == AbstractInsnNode.LINE) {
+        LineNumberNode lineNode = (LineNumberNode) current;
+        lines.add(lineNode.line);
+      }
+      current = current.getNext();
+    }
+    return lines;
+  }
+
   /** Wraps ASM's {@link org.objectweb.asm.Type} with associated generic types */
   public static class Type {
     private final org.objectweb.asm.Type mainType;

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/CapturedContextInstrumentor.java

@@ -909,7 +909,7 @@ private void collectLocalVariables(AbstractInsnNode location, InsnList insnList)
     }
 
     if (methodNode.localVariables == null || methodNode.localVariables.isEmpty()) {
-      if (!Config.get().isDynamicInstrumentationInstrumentTheWorld()) {
+      if (Config.get().getDynamicInstrumentationInstrumentTheWorld() == null) {
         reportWarning("Missing local variable debug info");
       }
       // no local variables info - bail out

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/ProbeStatusSink.java

@@ -58,7 +58,7 @@ public ProbeStatusSink(Config config, String diagnosticsEndpoint, boolean useMul
     this.interval = Duration.ofSeconds(config.getDynamicInstrumentationDiagnosticsInterval());
     this.batchSize = config.getDynamicInstrumentationUploadBatchSize();
     this.queue = new ArrayBlockingQueue<>(2 * this.batchSize);
-    this.isInstrumentTheWorld = config.isDynamicInstrumentationInstrumentTheWorld();
+    this.isInstrumentTheWorld = config.getDynamicInstrumentationInstrumentTheWorld() != null;
   }
 
   public void stop() {

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/uploader/BatchUploader.java

@@ -128,7 +128,7 @@ public BatchUploader(Config config, String endpoint, RetryPolicy retryPolicy) {
       RetryPolicy retryPolicy,
       String containerId,
       String entityId) {
-    instrumentTheWorld = config.isDynamicInstrumentationInstrumentTheWorld();
+    instrumentTheWorld = config.getDynamicInstrumentationInstrumentTheWorld() != null;
     if (endpoint == null || endpoint.length() == 0) {
       throw new IllegalArgumentException("Endpoint url is empty");
     }