Fix false disk-full read-only during concurrent snapshot directory de…

iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java

@@ -112,6 +112,8 @@ public final class UtilMessages {
 
   public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY =
       "All folders are full, change system mode to read-only.";
+  public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE =
+      "Cannot select folder but disk still has available space, will not change to read-only.";
   public static final String FAILED_TO_PROCESS_FOLDER = "Failed to process folder {}";
   public static final String FAILED_TO_READ_FILE_STORE_PATH =
       "Failed to read file store path '{}'";

iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java

@@ -110,6 +110,8 @@ public final class UtilMessages {
 
   public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY =
       "所有文件夹空间均已耗尽，系统切换为只读模式。";
+  public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE =
+      "无法选择文件夹但磁盘仍有可用空间，不切换为只读模式。";
   public static final String FAILED_TO_PROCESS_FOLDER = "处理文件夹 {} 失败";
   public static final String FAILED_TO_READ_FILE_STORE_PATH =
       "读取文件存储路径 '{}' 失败";

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java

@@ -88,9 +88,7 @@ public FolderManager(List<String> folders, DirectoryStrategyType type)
       this.selectStrategy.setFolders(folders);
       this.selectStrategy.setFoldersStates(foldersStates);
     } catch (DiskSpaceInsufficientException e) {
-      logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
-      CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
-      CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+      changeToReadOnlyIfDiskFull(e);
       throw e;
     }
   }
@@ -104,9 +102,7 @@ public String getNextFolder() throws DiskSpaceInsufficientException {
     try {
       return folders.get(selectStrategy.nextFolderIndex());
     } catch (DiskSpaceInsufficientException e) {
-      logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
-      CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
-      CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+      changeToReadOnlyIfDiskFull(e);
       throw e;
     }
   }
@@ -118,6 +114,24 @@ boolean hasHealthyFolder() {
                 foldersStates.getOrDefault(folder, FolderState.ABNORMAL) == FolderState.HEALTHY);
   }
 
+  private boolean hasFolderWithAvailableDiskSpace() {
+    return folders.stream()
+        .anyMatch(
+            folder ->
+                foldersStates.getOrDefault(folder, FolderState.ABNORMAL) == FolderState.HEALTHY
+                    && JVMCommonUtils.hasSpace(folder));
+  }
+
+  private void changeToReadOnlyIfDiskFull(DiskSpaceInsufficientException e) {
+    if (!hasFolderWithAvailableDiskSpace()) {
+      logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
+      CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+      CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+    } else {
+      logger.warn(UtilMessages.CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE, e);
+    }
+  }
+
   @FunctionalInterface
   public interface ThrowingFunction<T, R, E extends Exception> {
     R apply(T t) throws E;
@@ -135,9 +149,7 @@ public <T, E extends Exception> T getNextWithRetry(ThrowingFunction<String, T, E
       try {
         folder = folders.get(selectStrategy.nextFolderIndex());
       } catch (DiskSpaceInsufficientException e) {
-        logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
-        CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
-        CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+        changeToReadOnlyIfDiskFull(e);
         throw e;
       }
       try {

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java

@@ -120,8 +120,7 @@ private void refreshOccupiedSpace() {
       try {
         cachedOccupiedSpace[i] = JVMCommonUtils.getOccupiedSpace(folder);
       } catch (IOException | UncheckedIOException e) {
-        LOGGER.error(UtilMessages.CANNOT_CALCULATE_OCCUPIED_SPACE, folder, e);
-        cachedOccupiedSpace[i] = Long.MAX_VALUE;
+        LOGGER.warn(UtilMessages.CANNOT_CALCULATE_OCCUPIED_SPACE, folder, e);
       }
     }
     selectionsSinceRefresh = 0;

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java

@@ -28,10 +28,12 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.stream.Stream;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 
 public class JVMCommonUtils {
 
@@ -110,15 +112,23 @@ public static long getOccupiedSpace(String folderPath) throws IOException {
     if (!Files.exists(folder)) {
       return 0;
     }
-    try (Stream<Path> s = Files.walk(folder)) {
-      return s.filter(p -> p.toFile().isFile())
-          .mapToLong(
-              p -> {
-                File file = p.toFile();
-                return file.exists() ? file.length() : 0L;
-              })
-          .sum();
-    }
+    final long[] occupiedSpace = {0L};
+    Files.walkFileTree(
+        folder,
+        new SimpleFileVisitor<Path>() {
+          @Override
+          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+            occupiedSpace[0] += attrs.size();
+            return FileVisitResult.CONTINUE;
+          }
+
+          @Override
+          public FileVisitResult visitFileFailed(Path file, IOException exc) {
+            // A file or directory may be deleted concurrently during traversal; ignore it.
+            return FileVisitResult.CONTINUE;
+          }
+        });
+    return occupiedSpace[0];
   }
 
   public static int getCpuCores() {

iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java

@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.commons.disk;
 
+import org.apache.iotdb.commons.cluster.NodeStatus;
 import org.apache.iotdb.commons.conf.CommonDescriptor;
 import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
 import org.apache.iotdb.commons.disk.strategy.MinFolderOccupiedSpaceFirstStrategy;
@@ -34,8 +35,10 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Stream;
 
 import static org.junit.Assert.assertEquals;
@@ -124,4 +127,62 @@ public void cachesOccupiedSpaceAndRefreshesAgainstRealFiles()
     // and correctly avoids the now-largest folder 0, picking the least occupied folder 1.
     assertEquals(1, strategy.nextFolderIndex());
   }
+
+  @Test
+  public void getNextFolderDoesNotEnterReadOnlyWhenDiskHasSpace() throws Exception {
+    Path snapshotRoot = Files.createTempDirectory("snapshot-root-");
+    tempDirs.add(snapshotRoot);
+    Path subDir = snapshotRoot.resolve("tod_sod0-71");
+    Files.createDirectories(subDir);
+    writeFileInPath(subDir, "a.bin", 1024);
+
+    List<String> singleFolder = Collections.singletonList(snapshotRoot.toFile().getAbsolutePath());
+    CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Running);
+
+    AtomicBoolean stop = new AtomicBoolean(false);
+    Thread deleter =
+        new Thread(
+            () -> {
+              while (!stop.get()) {
+                try {
+                  deleteRecursively(subDir);
+                  Files.createDirectories(subDir);
+                  writeFileInPath(subDir, "temp.bin", 512);
+                  Thread.sleep(1);
+                } catch (Exception ignored) {
+                  // Concurrent create/delete is expected in this test.
+                }
+              }
+            });
+    deleter.start();
+
+    try {
+      FolderManager folderManager =
+          new FolderManager(
+              singleFolder, DirectoryStrategyType.MIN_FOLDER_OCCUPIED_SPACE_FIRST_STRATEGY);
+      for (int i = 0; i < 100; i++) {
+        assertEquals(
+            NodeStatus.Running, CommonDescriptor.getInstance().getConfig().getNodeStatus());
+        assertEquals(singleFolder.get(0), folderManager.getNextFolder());
+      }
+    } finally {
+      stop.set(true);
+      deleter.join(5000);
+    }
+  }
+
+  private void writeFileInPath(Path folder, String name, int sizeBytes) throws IOException {
+    byte[] payload = new byte[sizeBytes];
+    Arrays.fill(payload, (byte) 1);
+    Files.write(folder.resolve(name), payload);
+  }
+
+  private static void deleteRecursively(Path path) throws IOException {
+    if (!Files.exists(path)) {
+      return;
+    }
+    try (Stream<Path> walk = Files.walk(path)) {
+      walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+    }
+  }
 }

iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java

@@ -28,6 +28,10 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Stream;
 
 public class JVMCommonUtilsTest {
 
@@ -66,4 +70,48 @@ public void getOccupiedSpaceSumsFileSizes() throws IOException {
     Assert.assertEquals(
         2L * payload.length, JVMCommonUtils.getOccupiedSpace(dir.getAbsolutePath()));
   }
+
+  @Test
+  public void getOccupiedSpaceIgnoresConcurrentlyDeletedEntries() throws Exception {
+    File snapshotRoot = tempFolder.newFolder("snapshot");
+    File subDir = new File(snapshotRoot, "tod_sod0-71");
+    Assert.assertTrue(subDir.mkdirs());
+    byte[] payload = "snapshot-data".getBytes(StandardCharsets.UTF_8);
+    Files.write(new File(subDir, "a.bin").toPath(), payload);
+    Files.write(new File(snapshotRoot, "other.bin").toPath(), payload);
+
+    AtomicBoolean stop = new AtomicBoolean(false);
+    Thread deleter =
+        new Thread(
+            () -> {
+              while (!stop.get()) {
+                try {
+                  deleteRecursively(subDir.toPath());
+                  Files.createDirectories(subDir.toPath());
+                  Files.write(new File(subDir, "temp.bin").toPath(), payload);
+                } catch (IOException ignored) {
+                  // Concurrent create/delete is expected in this test.
+                }
+              }
+            });
+    deleter.start();
+
+    try {
+      for (int i = 0; i < 200; i++) {
+        JVMCommonUtils.getOccupiedSpace(snapshotRoot.getAbsolutePath());
+      }
+    } finally {
+      stop.set(true);
+      deleter.join(5000);
+    }
+  }
+
+  private static void deleteRecursively(Path path) throws IOException {
+    if (!Files.exists(path)) {
+      return;
+    }
+    try (Stream<Path> walk = Files.walk(path)) {
+      walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+    }
+  }
 }