diff --git a/gradle/configure_tests.gradle b/gradle/configure_tests.gradle index 076acd7e8cd..3830b063bfb 100644 --- a/gradle/configure_tests.gradle +++ b/gradle/configure_tests.gradle @@ -1,6 +1,8 @@ import java.time.Duration import java.time.temporal.ChronoUnit +apply from: "$rootDir/gradle/dump_hanging_test.gradle" + def isTestingInstrumentation(Project project) { return [ "junit-4.10", @@ -126,7 +128,7 @@ if (!project.property("activePartition")) { } } -tasks.withType(Test) { +tasks.withType(Test).configureEach { // https://docs.gradle.com/develocity/flaky-test-detection/ // https://docs.gradle.com/develocity/gradle-plugin/current/#test_retry develocity.testRetry { diff --git a/gradle/dump_hanging_test.gradle b/gradle/dump_hanging_test.gradle new file mode 100644 index 00000000000..42a8909cbdb --- /dev/null +++ b/gradle/dump_hanging_test.gradle @@ -0,0 +1,75 @@ +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +// Schedule thread and heap dumps collection near test timeout. +tasks.withType(Test).configureEach { + doFirst { + def isCI = System.getenv('CI') != null + + // Use Gradle's build dir and adjust for CI artifacts collection if needed. + String buildDir = layout.buildDirectory.asFile.get().absolutePath + if (isCI) { + // Move reports into the folder collected by the collect_reports.sh script. + buildDir = buildDir.replace('dd-trace-java/dd-java-agent', 'dd-trace-java/workspace/dd-java-agent') + } + + def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> + Thread t = new Thread(r, 'dump-scheduler') + t.daemon = true + t + }) + + // Calculate delay for taking dumps as test timeout minus 2 minutes, but no less than 5 minutes. + def delay = timeout.get().minusMinutes(2) + long delayMinutes = Math.max(5L, delay.toMinutes()) + + // Schedule the dump job. + def future = scheduler.schedule({ + try { + def dumpDir = new File(buildDir, 'dumps') + dumpDir.mkdirs() + + // Collect PIDs of all Java processes. + def jvmProcesses = 'jcmd -l'.execute().text.readLines() + + jvmProcesses.each { line -> + // Only process 'Gradle test executors'. + if (!line.contains('Gradle Test Executor')) return + + def pid = line.substring(0, line.indexOf(' ')) + + // Collect thread dump. + def threadDumpFile = new File(dumpDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log") + new ProcessBuilder('jcmd', pid, 'Thread.print', '-l') + .redirectErrorStream(true) + .redirectOutput(threadDumpFile) + .start() + .waitFor() + + // Collect heap dump. + def heapDumpFile = "${dumpDir.absolutePath}/${pid}-heap-dump-${System.currentTimeMillis()}.hprof" + def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}" + cmd.execute().waitFor() + } + } catch (Throwable e) { + logger.warn("Dumping failed: ${e.message}") + } + finally { + scheduler.shutdown() + } + }, delayMinutes, TimeUnit.MINUTES) + + // Store handles for cancellation in doLast. + ext.dumpFuture = future + ext.dumpScheduler = scheduler + } + + doLast { + // Cancel if the task finished before the scheduled dump. + try { + ext.dumpFuture?.cancel(false) + } finally { + ext.dumpScheduler?.shutdownNow() + } + } +}