Upgrade Shadow plugin to 9.4.2

build.gradle.kts

@@ -17,7 +17,7 @@ plugins {
   id("com.github.spotbugs") version "6.5.5"
   id("de.thetaphi.forbiddenapis") version "3.10"
   id("io.github.gradle-nexus.publish-plugin") version "2.0.0"
-  id("com.gradleup.shadow") version "8.3.9" apply false
+  id("com.gradleup.shadow") version "9.4.2" apply false
   id("me.champeau.jmh") version "0.7.3" apply false
   id("org.gradle.playframework") version "0.16.0" apply false
 }

buildSrc/call-site-instrumentation-plugin/build.gradle.kts

@@ -1,7 +1,7 @@
 plugins {
   java
   id("com.diffplug.spotless") version "8.4.0"
-  id("com.gradleup.shadow") version "8.3.9"
+  id("com.gradleup.shadow") version "9.4.2"
 }
 
 java {
@@ -69,7 +69,13 @@ tasks {
   }
 
   shadowJar {
+    duplicatesStrategy = DuplicatesStrategy.INCLUDE
     mergeServiceFiles()
+    // Service descriptors are intentionally merged by mergeServiceFiles(); let
+    // duplicate service entries reach that transformer instead of failing first.
+    filesNotMatching("META-INF/services/**") {
+      duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    }
     manifest {
       attributes(mapOf("Main-Class" to "datadog.trace.plugin.csi.PluginApplication"))
     }

buildSrc/modifiable-config-agent/build.gradle.kts

@@ -1,6 +1,6 @@
 plugins {
   java
-  id("com.gradleup.shadow") version "8.3.9"
+  id("com.gradleup.shadow") version "9.4.2"
 }
 
 java {

dd-java-agent/agent-aiguard/build.gradle

@@ -34,4 +34,3 @@ tasks.named("shadowJar", ShadowJar) {
 tasks.named("jar", Jar) {
   archiveClassifier = 'unbundled'
 }
-

dd-java-agent/benchmark-integration/build.gradle

@@ -1,26 +1,3 @@
-buildscript {
-  repositories {
-    mavenLocal()
-    if (project.rootProject.hasProperty("gradlePluginProxy")) {
-      maven {
-        url project.rootProject.property("gradlePluginProxy")
-        allowInsecureProtocol = true
-      }
-    }
-    if (project.rootProject.hasProperty("mavenRepositoryProxy")) {
-      maven {
-        url project.rootProject.property("mavenRepositoryProxy")
-        allowInsecureProtocol = true
-      }
-    }
-    gradlePluginPortal()
-    mavenCentral()
-  }
-  dependencies {
-    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.1'
-  }
-}
-
 apply from: "$rootDir/gradle/java.gradle"
 
 description = 'Integration Level Agent benchmarks.'

dd-java-agent/build.gradle

@@ -81,8 +81,24 @@ dependencies {
 def generalShadowJarConfig(ShadowJar shadowJarTask) {
   shadowJarTask.with {
     mergeServiceFiles()
+    addMultiReleaseAttribute = false
 
     duplicatesStrategy = DuplicatesStrategy.FAIL
+    // Service descriptors are intentionally merged by mergeServiceFiles(); let
+    // duplicate service entries reach that transformer instead of failing first.
+    filesMatching('META-INF/services/**') {
+      duplicatesStrategy = DuplicatesStrategy.INCLUDE
+    }
+    // Vendored dependencies often repeat license/notice metadata. Keep one copy
+    // while still failing on unexpected duplicate runtime resources and classes.
+    filesMatching([
+      'META-INF/LICENSE*',
+      'META-INF/NOTICE*',
+      'META-INF/AL2.0',
+      'META-INF/LGPL2.1',
+    ]) {
+      duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    }
 
     // Remove some cruft from the final jar.
     // These patterns should NOT include **/META-INF/maven/**/pom.properties, which is
@@ -99,6 +115,7 @@ def generalShadowJarConfig(ShadowJar shadowJarTask) {
     exclude '**/inst/META-INF/versions/**'
     exclude '**/META-INF/versions/*/org/yaml/**'
     exclude '**/package.html'
+    exclude '**/about.html'
 
     // Used to generate Java code during build, no need to include original file
     exclude '**/*.trie'
@@ -257,7 +274,7 @@ includeSubprojShadowJar(project(':products:metrics:metrics-lib'), 'metrics', inc
 includeSubprojShadowJar(project(':products:feature-flagging:feature-flagging-agent'), 'feature-flagging', includedJarFileTree)
 
 def sharedShadowJar = tasks.register('sharedShadowJar', ShadowJar) {
-  it.configurations = [project.configurations.sharedShadowInclude]
+  it.configurations.add(project.configurations.named('sharedShadowInclude'))
   // Put the jar in a different directory so we don't overwrite the normal shadow jar and
   // break caching, and also to not interfere with CI scripts that copy everything in the
   // libs directory
@@ -275,16 +292,16 @@ def sharedShadowJar = tasks.register('sharedShadowJar', ShadowJar) {
     exclude(project(':utils:time-utils'))
     exclude(project(':products:metrics:metrics-api'))
     exclude(project(':products:metrics:metrics-agent'))
-    exclude(dependency('org.slf4j::'))
+    exclude(dependency('org.slf4j:.*:.*'))
     // use dd-instrument-java's embedded copy of asm
-    exclude(dependency('org.ow2.asm:asm:'))
+    exclude(dependency('org.ow2.asm:asm:.*'))
   }
 }
 includeShadowJar(sharedShadowJar, 'shared', includedJarFileTree)
 
 // place the tracer in its own shadow jar separate to instrumentation
 def traceShadowJar = tasks.register('traceShadowJar', ShadowJar) {
-  it.configurations = [project.configurations.traceShadowInclude]
+  it.configurations.add(project.configurations.named('traceShadowInclude'))
   it.destinationDirectory.set(project.layout.buildDirectory.dir("trace-lib"))
   it.archiveClassifier = 'trace'
   it.dependencies deps.excludeShared
@@ -299,7 +316,10 @@ tasks.named("shadowJar", ShadowJar) {
 
   generalShadowJarConfig(it)
 
-  configurations = [project.configurations.shadowInclude]
+  // The default shadowJar task has a runtimeClasspath convention. Replace it
+  // instead of adding to it, otherwise runtimeClasspath would be bundled too.
+  configurations.empty()
+  configurations.add(project.configurations.named('shadowInclude'))
 
   archiveClassifier = ''
 

dd-java-agent/ddprof-lib/build.gradle

@@ -16,10 +16,8 @@ dependencies {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  dependencies {
-    deps.excludeShared
-    exclude '**/*.debug'
-  }
+  dependencies deps.excludeShared
+  exclude '**/*.debug'
   archiveClassifier = 'all'
   include {
     def rslt = false

dd-java-agent/instrumentation/build.gradle

@@ -120,13 +120,37 @@ if (project.gradle.startParameter.taskNames.any { it.endsWith("generateMuzzleRep
 
 tasks.named('shadowJar', ShadowJar) {
   duplicatesStrategy = DuplicatesStrategy.FAIL
+  // Shadow 8.x silently wrote duplicate class (130+) entries into the instrumentation jar.
+  // Those duplicate entries did not survive as duplicates in the final agent jar,
+  // because `:dd-java-agent:expandAgentShadowJarInst` expands the instrumentation jar
+  // with a `Sync` task, and the specific `inst` expansion keeps the first duplicate path.
+  // (explicit `duplicatesStrategy = DuplicatesStrategy.EXCLUDE` on the `inst` prefix).
+  //
+  // Shadow 9 refactored deeply it's "copy" pipeline via https://github.com/GradleUp/shadow/pull/1233
+  // by `Project.zipTree` and can now honor `DuplicatesStrategy`, see in particular
+  // * https://github.com/GradleUp/shadow/issues/1223 - Duplicates from jars are not handled per duplicatesStrategy
+  // * https://github.com/GradleUp/shadow/issues/488 - DuplicatesStrategy.FAIL doesn't work with transform
+  //
+  // Keeping `duplicatesStrategy = FAIL` only, Shadow 9 now fails while creating the instrumentation jar.
+  filesMatching([
+    // agent-installer also publishes this Java 11 source-set output at runtime.
+    'datadog/trace/agent/tooling/bytebuddy/DDJava9ClassFileTransformer*.class',
+    // agent-installer also publishes this Java 25 source-set output at runtime.
+    'datadog/trace/agent/tooling/servicediscovery/MemFDUnixWriterFFM*.class',
+    // aggregate instrumentation includes some classes directly and via jars.
+    'datadog/trace/instrumentation/**/*.class',
+    // same aggregate duplication, for Java 11 exception profiling advice.
+    'datadog/exceptions/instrumentation/**/*.class',
+  ]) {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+  }
   dependencies {
     // the tracer is now in a separate shadow jar
     exclude(project(":dd-trace-core"))
     exclude(dependency('com.datadoghq:sketches-java'))
     exclude(dependency('com.google.re2j:re2j'))
+    deps.excludeShared.execute(it)
   }
-  dependencies deps.excludeShared
 }
 
 // temporary config to add slf4j-simple so we get logging from instrumenters while indexing

dd-java-agent/instrumentation/liberty/liberty-23.0/build.gradle

@@ -137,7 +137,10 @@ tasks.named('filterLogbackClassic', Sync) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.shadow]
+  // The default shadowJar task has a runtimeClasspath convention. Replace it
+  // instead of adding to it; this jar is intentionally built from the shadow configuration only.
+  configurations.empty()
+  configurations.add(project.configurations.named('shadow'))
   zip64 = true
 
   archiveFileName = 'openliberty-shadow.jar'

dd-java-agent/instrumentation/servlet/jakarta-servlet-5.0/build.gradle

@@ -37,7 +37,7 @@ tasks.register('relocatedJavaxJar', ShadowJar) {
 
   archiveClassifier.set('relocated-javax')
 
-  configurations = [project.configurations.javaxClassesToRelocate]
+  configurations.add(project.configurations.named('javaxClassesToRelocate'))
 
   include '**/*.jar'
   include '**/Servlet31InputStreamWrapper.class'

dd-java-agent/testing/build.gradle

@@ -67,9 +67,9 @@ tasks.named("shadowJar", ShadowJar) {
   // Only bundle jetty dependencies into the jar (relocated)
   // All other dependencies remain as transitive dependencies
   dependencies {
-    include(dependency {
+    include {
       it.moduleGroup == 'org.eclipse.jetty' || it.moduleGroup == 'org.eclipse.jetty.http2'
-    })
+    }
   }
 
   // Relocate jetty classes to avoid conflicts with instrumented versions

dd-smoke-tests/appsec/springboot-graphql/build.gradle

@@ -1,4 +1,5 @@
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+import com.github.jengelman.gradle.plugins.shadow.transformers.PropertiesFileTransformer
 
 plugins {
   id 'com.gradleup.shadow'
@@ -16,8 +17,23 @@ jar {
 }
 
 shadowJar {
-  mergeServiceFiles {
-    include 'META-INF/spring.*'
+  duplicatesStrategy = DuplicatesStrategy.INCLUDE
+  mergeServiceFiles()
+  append 'META-INF/spring.handlers'
+  append 'META-INF/spring.schemas'
+  append 'META-INF/spring.tooling'
+  transform(PropertiesFileTransformer) {
+    paths = ['META-INF/spring.factories']
+    mergeStrategy = "append"
+  }
+  filesNotMatching([
+    'META-INF/services/**',
+    'META-INF/spring.handlers',
+    'META-INF/spring.schemas',
+    'META-INF/spring.tooling',
+    'META-INF/spring.factories',
+  ]) {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
   }
 }
 

dd-smoke-tests/custom-systemloader/build.gradle

@@ -14,7 +14,7 @@ tasks.named("jar", Jar) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.runtimeClasspath]
+  configurations.add(project.configurations.named('runtimeClasspath'))
 }
 
 dependencies {

dd-smoke-tests/field-injection/build.gradle

@@ -14,7 +14,7 @@ tasks.named("jar", Jar) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.runtimeClasspath]
+  configurations.add(project.configurations.named('runtimeClasspath'))
 }
 
 dependencies {

dd-smoke-tests/grpc-1.5/build.gradle

@@ -39,8 +39,12 @@ protobuf {
 }
 
 tasks.withType(ShadowJar).configureEach {
+  duplicatesStrategy = DuplicatesStrategy.INCLUDE
   archiveClassifier.set("fat")
   mergeServiceFiles()
+  filesNotMatching('META-INF/services/**') {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+  }
 }
 
 tasks.withType(Test).configureEach {

dd-smoke-tests/iast-propagation/build.gradle

@@ -30,7 +30,7 @@ tasks.named("jar", Jar) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.runtimeClasspath]
+  configurations.add(project.configurations.named('runtimeClasspath'))
 }
 
 dependencies {

dd-smoke-tests/log-injection/build.gradle

@@ -178,7 +178,9 @@ def generateTestingJar(String interfaceName, String backend, List<NamedDomainObj
 
     archiveBaseName.set(name)
 
-    configurations = [project.configurations.named('runtimeClasspath').get()] + configurationProviders.collect { it.get() }
+    configurations = providers.provider {
+      [project.configurations.named('runtimeClasspath').get()] + configurationProviders.collect { it.get() }
+    }
   }
 }
 
@@ -392,4 +394,3 @@ tasks.withType(Test).configureEach {
     .get()
 }
 
-

dd-smoke-tests/spring-boot-2.3-webmvc-jetty/build.gradle

@@ -17,7 +17,8 @@ tasks.named("jar", Jar) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.runtimeClasspath]
+  configurations.add(project.configurations.named('runtimeClasspath'))
+  duplicatesStrategy = DuplicatesStrategy.INCLUDE
   mergeServiceFiles()
   append 'META-INF/spring.handlers'
   append 'META-INF/spring.schemas'
@@ -26,6 +27,15 @@ tasks.named("shadowJar", ShadowJar) {
     paths = ['META-INF/spring.factories']
     mergeStrategy = "append"
   }
+  filesNotMatching([
+    'META-INF/services/**',
+    'META-INF/spring.handlers',
+    'META-INF/spring.schemas',
+    'META-INF/spring.tooling',
+    'META-INF/spring.factories',
+  ]) {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+  }
 }
 
 dependencies {

dd-smoke-tests/spring-boot-2.4-webflux/build.gradle

@@ -16,7 +16,7 @@ tasks.named("jar", Jar) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.runtimeClasspath]
+  configurations.add(project.configurations.named('runtimeClasspath'))
 }
 
 dependencies {

dd-smoke-tests/spring-boot-2.5-webflux/build.gradle

@@ -16,7 +16,7 @@ tasks.named("jar", Jar) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.runtimeClasspath]
+  configurations.add(project.configurations.named('runtimeClasspath'))
 }
 
 dependencies {

dd-smoke-tests/spring-boot-2.6-webflux/build.gradle

@@ -17,7 +17,8 @@ tasks.named("jar", Jar) {
 }
 
 tasks.named("shadowJar", ShadowJar) {
-  configurations = [project.configurations.runtimeClasspath]
+  configurations.add(project.configurations.named('runtimeClasspath'))
+  duplicatesStrategy = DuplicatesStrategy.INCLUDE
   mergeServiceFiles()
   append 'META-INF/spring.handlers'
   append 'META-INF/spring.schemas'
@@ -26,6 +27,15 @@ tasks.named("shadowJar", ShadowJar) {
     paths = ['META-INF/spring.factories']
     mergeStrategy = "append"
   }
+  filesNotMatching([
+    'META-INF/services/**',
+    'META-INF/spring.handlers',
+    'META-INF/spring.schemas',
+    'META-INF/spring.tooling',
+    'META-INF/spring.factories',
+  ]) {
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+  }
 }
 
 dependencies {