diff --git a/src/org/labkey/test/TestFileUtils.java b/src/org/labkey/test/TestFileUtils.java index d73ad973a1..23b77c54a7 100644 --- a/src/org/labkey/test/TestFileUtils.java +++ b/src/org/labkey/test/TestFileUtils.java @@ -21,6 +21,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.pdfbox.Loader; @@ -40,6 +41,7 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.util.FileUtil; import org.jetbrains.annotations.Nullable; +import org.labkey.api.util.StringUtilsLabKey; import org.openqa.selenium.NotFoundException; import java.io.BufferedInputStream; @@ -57,6 +59,7 @@ import java.io.Writer; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.nio.file.AccessDeniedException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -71,10 +74,13 @@ import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import static org.labkey.api.util.DebugInfoDumper.dumpHeap; +import static org.labkey.test.WebDriverWrapper.sleep; import static org.labkey.test.util.TestDataGenerator.CHARSET_STRING; import static org.labkey.test.util.TestDataGenerator.randomInt; import static org.labkey.test.util.TestDataGenerator.randomName; @@ -445,6 +451,80 @@ public static void deleteDir(File dir) } } + /** + * Deletes a directory and all its contents, retrying up to 10 times with a 10-second delay between attempts. + *

+ * Before each attempt, the directory and all its children are marked writable to handle read-only files or + * directories. This is primarily intended to work around Windows file-locking issues where an external process + * may hold a lock on the directory or its contents. + *

+ * On the final failed attempt, a heap dump is captured for diagnostics if running on TeamCity. The list of running + * processes is also logged to help identify what may be holding the lock. + * + * @param dir the directory to delete + * @throws Exception if an unexpected error occurs + */ + public static void deleteDirWithRetry(File dir) throws Exception + { + // Sometimes on Windows the directory could be locked, maybe by an external process, or the child directory is + // readonly. Use a retry mechanism to set the writeable flag and then try to delete the parent directory. + for (int attempt = 1; attempt <= 10; attempt++) { + try { + dir.setWritable(true, false); + + // Wrap in a try to close the stream. + try (Stream files = Files.walk(dir.toPath())) { + files.forEach(p -> p.toFile().setWritable(true, false)); + } + + FileUtils.deleteDirectory(dir); + LOG.info(String.format("Deletion of directory %s was successful.", dir)); + break; + } catch (IOException ioException) { + LOG.warn(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", + dir, ioException.getMessage(), attempt)); + if (attempt == 10) { + + if (TestProperties.isTestRunningOnTeamCity()) { + LOG.info("Dump the heap."); + dumpHeap(); + } + + ProcessBuilder pb; + if (SystemUtils.IS_OS_WINDOWS) { + pb = new ProcessBuilder("tasklist"); + } + else { + pb = new ProcessBuilder("ps", "-ef"); + } + + try { + LOG.info("Lock diagnostic..."); + pb.redirectErrorStream(true); + + // Tried to use "try (Process p = pb.start()) {" without the finally block but our build + // system didn't like that and complained that Process doesn't implement closeable (it does). + Process p = pb.start(); + try { + String output = new String(p.getInputStream().readAllBytes(), StringUtilsLabKey.DEFAULT_CHARSET); + LOG.info("Running processes:\n" + output); + } + finally { + // Don't leak the process resource. + p.destroy(); + } + + } catch (IOException diagnosticException) { + LOG.warn("Failed to run lock diagnostic: " + diagnosticException.getMessage()); + } + throw ioException; + } + sleep(10_000); + } + } + + } + private static void checkFileLocation(File file) { try diff --git a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java index d0979ba985..48763c9eb8 100644 --- a/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java +++ b/src/org/labkey/test/tests/assay/AssayTransformMissingParentDirTest.java @@ -1,6 +1,5 @@ package org.labkey.test.tests.assay; -import org.apache.commons.io.FileUtils; import org.junit.Test; import org.junit.experimental.categories.Category; import org.labkey.api.util.FileUtil; @@ -15,8 +14,6 @@ import org.labkey.test.params.assay.GeneralAssayDesign; import java.io.File; -import java.io.IOException; -import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Path; @@ -48,29 +45,8 @@ public void testMissingParentDirectoryRegression() throws Exception assayDesignerPage.addTransformScript(transformFile); assayDesignerPage.clickSave(); - // Now delete the parent dir to ensure we handle it reasonably. On Windows something locks the directory, maybe - // an external process. If that happens sleep for a second and try again. - for (int attempt = 1; attempt <= 10; attempt++) { - try - { - FileUtils.deleteDirectory(parentDir.toFile()); - log(String.format("Deletion of directory %s was successful.", parentDir)); - break; - } catch (AccessDeniedException deniedException) { - // Yes I know AccessDeniedException is a subset of an IOException, but I wanted to log explicitly a - // failure and retry because of an AccessDeniedException from some other IOException. - log(String.format("Access denied trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", - parentDir, deniedException.getMessage(), attempt)); - if (attempt == 10) throw deniedException; - sleep(10_000); - } - catch (IOException ioException) { - log(String.format("IOException trying to delete directory %s. Error: %s. Waiting 10s and retrying. Attempt %d of 10.", - parentDir, ioException.getMessage(), attempt)); - if (attempt == 10) throw ioException; - sleep(10_000); - } - } + // Now delete the parent dir to ensure we handle it reasonably + TestFileUtils.deleteDirWithRetry(parentDir.toFile()); // Attempt to import data and verify a reasonable error message is shown String importData = """