diff --git a/src/org/labkey/test/TestFileUtils.java b/src/org/labkey/test/TestFileUtils.java
index d73ad973a1..23b77c54a7 100644
--- a/src/org/labkey/test/TestFileUtils.java
+++ b/src/org/labkey/test/TestFileUtils.java
@@ -21,6 +21,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pdfbox.Loader;
@@ -40,6 +41,7 @@
import org.jetbrains.annotations.NotNull;
import org.labkey.api.util.FileUtil;
import org.jetbrains.annotations.Nullable;
+import org.labkey.api.util.StringUtilsLabKey;
import org.openqa.selenium.NotFoundException;
import java.io.BufferedInputStream;
@@ -57,6 +59,7 @@
import java.io.Writer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.nio.file.AccessDeniedException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -71,10 +74,13 @@
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import static org.labkey.api.util.DebugInfoDumper.dumpHeap;
+import static org.labkey.test.WebDriverWrapper.sleep;
import static org.labkey.test.util.TestDataGenerator.CHARSET_STRING;
import static org.labkey.test.util.TestDataGenerator.randomInt;
import static org.labkey.test.util.TestDataGenerator.randomName;
@@ -445,6 +451,80 @@ public static void deleteDir(File dir)
}
}
+ /**
+ * Deletes a directory and all its contents, retrying up to 10 times with a 10-second delay between attempts.
+ *
+ * Before each attempt, the directory and all its children are marked writable to handle read-only files or
+ * directories. This is primarily intended to work around Windows file-locking issues where an external process
+ * may hold a lock on the directory or its contents.
+ *
+ * On the final failed attempt, a heap dump is captured for diagnostics if running on TeamCity. The list of running
+ * processes is also logged to help identify what may be holding the lock.
+ *
+ * @param dir the directory to delete
+ * @throws Exception if an unexpected error occurs
+ */
+ public static void deleteDirWithRetry(File dir) throws Exception
+ {
+ // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is
+ // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory.
+ for (int attempt = 1; attempt <= 10; attempt++) {
+ try {
+ dir.setWritable(true, false);
+
+ // Wrap in a try to close the stream.
+ try (Stream files = Files.walk(dir.toPath())) {
+ files.forEach(p -> p.toFile().setWritable(true, false));
+ }
+
+ FileUtils.deleteDirectory(dir);
+ LOG.info(String.format("Deletion of directory %s was successful.", dir));
+ break;
+ } catch (IOException ioException) {
+ LOG.warn(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.",
+ dir, ioException.getMessage(), attempt));
+ if (attempt == 10) {
+
+ if (TestProperties.isTestRunningOnTeamCity()) {
+ LOG.info("Dump the heap.");
+ dumpHeap();
+ }
+
+ ProcessBuilder pb;
+ if (SystemUtils.IS_OS_WINDOWS) {
+ pb = new ProcessBuilder("tasklist");
+ }
+ else {
+ pb = new ProcessBuilder("ps", "-ef");
+ }
+
+ try {
+ LOG.info("Lock diagnostic...");
+ pb.redirectErrorStream(true);
+
+ // Tried to use "try (Process p = pb.start()) {" without the finally block but our build
+ // system didn't like that and complained that Process doesn't implement closeable (it does).
+ Process p = pb.start();
+ try {
+ String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET);
+ LOG.info("Running processes:\n" + output);
+ }
+ finally {
+ // Don't leak the process resource.
+ p.destroy();
+ }
+
+ } catch (IOException diagnosticException) {
+ LOG.warn("Failed to run lock diagnostic: " + diagnosticException.getMessage());
+ }
+ throw ioException;
+ }
+ sleep(10_000);
+ }
+ }
+
+ }
+
private static void checkFileLocation(File file)
{
try
diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java
index d0979ba985..48763c9eb8 100644
--- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java
+++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java
@@ -1,6 +1,5 @@
package org.labkey.test.tests.assay;
-import org.apache.commons.io.FileUtils;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.labkey.api.util.FileUtil;
@@ -15,8 +14,6 @@
import org.labkey.test.params.assay.GeneralAssayDesign;
import java.io.File;
-import java.io.IOException;
-import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -48,29 +45,8 @@ public void testMissingParentDirectoryRegression() throws Exception
assayDesignerPage.addTransformScript(transformFile);
assayDesignerPage.clickSave();
- // Now delete the parent dir to ensure we handle it reasonably. On Windows something locks the directory, maybe
- // an external process. If that happens sleep for a second and try again.
- for (int attempt = 1; attempt <= 10; attempt++) {
- try
- {
- FileUtils.deleteDirectory(parentDir.toFile());
- log(String.format("Deletion of directory %s was successful.", parentDir));
- break;
- } catch (AccessDeniedException deniedException) {
- // Yes I know AccessDeniedException is a subset of an IOException, but I wanted to log explicitly a
- // failure and retry because of an AccessDeniedException from some other IOException.
- log(String.format("Access denied trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.",
- parentDir, deniedException.getMessage(), attempt));
- if (attempt == 10) throw deniedException;
- sleep(10_000);
- }
- catch (IOException ioException) {
- log(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.",
- parentDir, ioException.getMessage(), attempt));
- if (attempt == 10) throw ioException;
- sleep(10_000);
- }
- }
+ // Now delete the parent dir to ensure we handle it reasonably
+ TestFileUtils.deleteDirWithRetry(parentDir.toFile());
// Attempt to import data and verify a reasonable error message is shown
String importData = """