diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java index 6008e507a9e7..0fbf9a941e5f 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java @@ -112,6 +112,8 @@ public final class UtilMessages { public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY = "All folders are full, change system mode to read-only."; + public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE = + "Cannot select folder but disk still has available space, will not change to read-only."; public static final String FAILED_TO_PROCESS_FOLDER = "Failed to process folder {}"; public static final String FAILED_TO_READ_FILE_STORE_PATH = "Failed to read file store path '{}'"; diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java index e1cfb9f7305c..6cdf3ffaacb7 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java @@ -110,6 +110,8 @@ public final class UtilMessages { public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY = "所有文件夹空间均已耗尽，系统切换为只读模式。"; + public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE = + "无法选择文件夹但磁盘仍有可用空间，不切换为只读模式。"; public static final String FAILED_TO_PROCESS_FOLDER = "处理文件夹 {} 失败"; public static final String FAILED_TO_READ_FILE_STORE_PATH = "读取文件存储路径 '{}' 失败"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java index 731dc66fa4a9..08e2b7872eff 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java @@ -88,9 +88,7 @@ public FolderManager(List folders, DirectoryStrategyType type) this.selectStrategy.setFolders(folders); this.selectStrategy.setFoldersStates(foldersStates); } catch (DiskSpaceInsufficientException e) { - logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); - CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL); + changeToReadOnlyIfDiskFull(e); throw e; } } @@ -104,9 +102,7 @@ public String getNextFolder() throws DiskSpaceInsufficientException { try { return folders.get(selectStrategy.nextFolderIndex()); } catch (DiskSpaceInsufficientException e) { - logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); - CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL); + changeToReadOnlyIfDiskFull(e); throw e; } } @@ -118,6 +114,24 @@ boolean hasHealthyFolder() { foldersStates.getOrDefault(folder, FolderState.ABNORMAL) == FolderState.HEALTHY); } + private boolean hasFolderWithAvailableDiskSpace() { + return folders.stream() + .anyMatch( + folder -> + foldersStates.getOrDefault(folder, FolderState.ABNORMAL) == FolderState.HEALTHY + && JVMCommonUtils.hasSpace(folder)); + } + + private void changeToReadOnlyIfDiskFull(DiskSpaceInsufficientException e) { + if (!hasFolderWithAvailableDiskSpace()) { + logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e); + CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); + CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL); + } else { + logger.warn(UtilMessages.CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE, e); + } + } + @FunctionalInterface public interface ThrowingFunction { R apply(T t) throws E; @@ -135,9 +149,7 @@ public T getNextWithRetry(ThrowingFunction s = Files.walk(folder)) { - return s.filter(p -> p.toFile().isFile()) - .mapToLong( - p -> { - File file = p.toFile(); - return file.exists() ? file.length() : 0L; - }) - .sum(); - } + final long[] occupiedSpace = {0L}; + Files.walkFileTree( + folder, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + occupiedSpace[0] += attrs.size(); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + // A file or directory may be deleted concurrently during traversal; ignore it. + return FileVisitResult.CONTINUE; + } + }); + return occupiedSpace[0]; } public static int getCpuCores() { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java index 6f4797233c56..d255d6fb7f48 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.commons.disk; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType; import org.apache.iotdb.commons.disk.strategy.MinFolderOccupiedSpaceFirstStrategy; @@ -34,8 +35,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -124,4 +127,62 @@ public void cachesOccupiedSpaceAndRefreshesAgainstRealFiles() // and correctly avoids the now-largest folder 0, picking the least occupied folder 1. assertEquals(1, strategy.nextFolderIndex()); } + + @Test + public void getNextFolderDoesNotEnterReadOnlyWhenDiskHasSpace() throws Exception { + Path snapshotRoot = Files.createTempDirectory("snapshot-root-"); + tempDirs.add(snapshotRoot); + Path subDir = snapshotRoot.resolve("tod_sod0-71"); + Files.createDirectories(subDir); + writeFileInPath(subDir, "a.bin", 1024); + + List singleFolder = Collections.singletonList(snapshotRoot.toFile().getAbsolutePath()); + CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Running); + + AtomicBoolean stop = new AtomicBoolean(false); + Thread deleter = + new Thread( + () -> { + while (!stop.get()) { + try { + deleteRecursively(subDir); + Files.createDirectories(subDir); + writeFileInPath(subDir, "temp.bin", 512); + Thread.sleep(1); + } catch (Exception ignored) { + // Concurrent create/delete is expected in this test. + } + } + }); + deleter.start(); + + try { + FolderManager folderManager = + new FolderManager( + singleFolder, DirectoryStrategyType.MIN_FOLDER_OCCUPIED_SPACE_FIRST_STRATEGY); + for (int i = 0; i < 100; i++) { + assertEquals( + NodeStatus.Running, CommonDescriptor.getInstance().getConfig().getNodeStatus()); + assertEquals(singleFolder.get(0), folderManager.getNextFolder()); + } + } finally { + stop.set(true); + deleter.join(5000); + } + } + + private void writeFileInPath(Path folder, String name, int sizeBytes) throws IOException { + byte[] payload = new byte[sizeBytes]; + Arrays.fill(payload, (byte) 1); + Files.write(folder.resolve(name), payload); + } + + private static void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java index d35d3a52ea29..e314093d9f41 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java @@ -28,6 +28,10 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; public class JVMCommonUtilsTest { @@ -66,4 +70,48 @@ public void getOccupiedSpaceSumsFileSizes() throws IOException { Assert.assertEquals( 2L * payload.length, JVMCommonUtils.getOccupiedSpace(dir.getAbsolutePath())); } + + @Test + public void getOccupiedSpaceIgnoresConcurrentlyDeletedEntries() throws Exception { + File snapshotRoot = tempFolder.newFolder("snapshot"); + File subDir = new File(snapshotRoot, "tod_sod0-71"); + Assert.assertTrue(subDir.mkdirs()); + byte[] payload = "snapshot-data".getBytes(StandardCharsets.UTF_8); + Files.write(new File(subDir, "a.bin").toPath(), payload); + Files.write(new File(snapshotRoot, "other.bin").toPath(), payload); + + AtomicBoolean stop = new AtomicBoolean(false); + Thread deleter = + new Thread( + () -> { + while (!stop.get()) { + try { + deleteRecursively(subDir.toPath()); + Files.createDirectories(subDir.toPath()); + Files.write(new File(subDir, "temp.bin").toPath(), payload); + } catch (IOException ignored) { + // Concurrent create/delete is expected in this test. + } + } + }); + deleter.start(); + + try { + for (int i = 0; i < 200; i++) { + JVMCommonUtils.getOccupiedSpace(snapshotRoot.getAbsolutePath()); + } + } finally { + stop.set(true); + deleter.join(5000); + } + } + + private static void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } }