From c42d345cea3398b9274a3c0972dcd612b2fcf47c Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Mon, 6 Jul 2026 10:51:48 +0800 Subject: [PATCH] Fix false disk-full read-only during concurrent snapshot directory deletion When receiving snapshot fragments, getOccupiedSpace walks the shared snapshot root while other regions may delete their staging subdirectories concurrently. NoSuchFileException was misinterpreted as folder full, triggering node read-only and failed region migration during datanode removal. --- .../iotdb/commons/i18n/UtilMessages.java | 2 + .../iotdb/commons/i18n/UtilMessages.java | 2 + .../iotdb/commons/disk/FolderManager.java | 30 ++++++--- .../MinFolderOccupiedSpaceFirstStrategy.java | 3 +- .../iotdb/commons/utils/JVMCommonUtils.java | 30 ++++++--- ...rOccupiedSpaceFirstStrategyRealFsTest.java | 61 +++++++++++++++++++ .../commons/utils/JVMCommonUtilsTest.java | 48 +++++++++++++++ 7 files changed, 155 insertions(+), 21 deletions(-) diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java index 6008e507a9e75..0fbf9a941e5f3 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java @@ -112,6 +112,8 @@ public final class UtilMessages { public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY = "All folders are full, change system mode to read-only."; + public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE = + "Cannot select folder but disk still has available space, will not change to read-only."; public static final String FAILED_TO_PROCESS_FOLDER = "Failed to process folder {}"; public static final String FAILED_TO_READ_FILE_STORE_PATH = "Failed to read file store path '{}'"; diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java index e1cfb9f7305c1..6cdf3ffaacb76 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java @@ -110,6 +110,8 @@ public final class UtilMessages { public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY = "所有文件夹空间均已耗尽，系统切换为只读模式。"; + public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE = + "无法选择文件夹但磁盘仍有可用空间，不切换为只读模式。"; public static final String FAILED_TO_PROCESS_FOLDER = "处理文件夹 {} 失败"; public static final String FAILED_TO_READ_FILE_STORE_PATH = "读取文件存储路径 '{}' 失败"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java index 731dc66fa4a99..08e2b7872efff 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java @@ -88,9 +88,7 @@ public FolderManager(List folders, DirectoryStrategyType type) this.selectStrategy.setFolders(folders); this.selectStrategy.setFoldersStates(foldersStates); } catch (DiskSpaceInsufficientException e) { - logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); - CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL); + changeToReadOnlyIfDiskFull(e); throw e; } } @@ -104,9 +102,7 @@ public String getNextFolder() throws DiskSpaceInsufficientException { try { return folders.get(selectStrategy.nextFolderIndex()); } catch (DiskSpaceInsufficientException e) { - logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); - CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL); + changeToReadOnlyIfDiskFull(e); throw e; } } @@ -118,6 +114,24 @@ boolean hasHealthyFolder() { foldersStates.getOrDefault(folder, FolderState.ABNORMAL) == FolderState.HEALTHY); } + private boolean hasFolderWithAvailableDiskSpace() { + return folders.stream() + .anyMatch( + folder -> + foldersStates.getOrDefault(folder, FolderState.ABNORMAL) == FolderState.HEALTHY + && JVMCommonUtils.hasSpace(folder)); + } + + private void changeToReadOnlyIfDiskFull(DiskSpaceInsufficientException e) { + if (!hasFolderWithAvailableDiskSpace()) { + logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e); + CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); + CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL); + } else { + logger.warn(UtilMessages.CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE, e); + } + } + @FunctionalInterface public interface ThrowingFunction { R apply(T t) throws E; @@ -135,9 +149,7 @@ public T getNextWithRetry(ThrowingFunction s = Files.walk(folder)) { - return s.filter(p -> p.toFile().isFile()) - .mapToLong( - p -> { - File file = p.toFile(); - return file.exists() ? file.length() : 0L; - }) - .sum(); - } + final long[] occupiedSpace = {0L}; + Files.walkFileTree( + folder, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + occupiedSpace[0] += attrs.size(); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + // A file or directory may be deleted concurrently during traversal; ignore it. + return FileVisitResult.CONTINUE; + } + }); + return occupiedSpace[0]; } public static int getCpuCores() { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java index 6f4797233c563..d255d6fb7f481 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.commons.disk; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType; import org.apache.iotdb.commons.disk.strategy.MinFolderOccupiedSpaceFirstStrategy; @@ -34,8 +35,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -124,4 +127,62 @@ public void cachesOccupiedSpaceAndRefreshesAgainstRealFiles() // and correctly avoids the now-largest folder 0, picking the least occupied folder 1. assertEquals(1, strategy.nextFolderIndex()); } + + @Test + public void getNextFolderDoesNotEnterReadOnlyWhenDiskHasSpace() throws Exception { + Path snapshotRoot = Files.createTempDirectory("snapshot-root-"); + tempDirs.add(snapshotRoot); + Path subDir = snapshotRoot.resolve("tod_sod0-71"); + Files.createDirectories(subDir); + writeFileInPath(subDir, "a.bin", 1024); + + List singleFolder = Collections.singletonList(snapshotRoot.toFile().getAbsolutePath()); + CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Running); + + AtomicBoolean stop = new AtomicBoolean(false); + Thread deleter = + new Thread( + () -> { + while (!stop.get()) { + try { + deleteRecursively(subDir); + Files.createDirectories(subDir); + writeFileInPath(subDir, "temp.bin", 512); + Thread.sleep(1); + } catch (Exception ignored) { + // Concurrent create/delete is expected in this test. + } + } + }); + deleter.start(); + + try { + FolderManager folderManager = + new FolderManager( + singleFolder, DirectoryStrategyType.MIN_FOLDER_OCCUPIED_SPACE_FIRST_STRATEGY); + for (int i = 0; i < 100; i++) { + assertEquals( + NodeStatus.Running, CommonDescriptor.getInstance().getConfig().getNodeStatus()); + assertEquals(singleFolder.get(0), folderManager.getNextFolder()); + } + } finally { + stop.set(true); + deleter.join(5000); + } + } + + private void writeFileInPath(Path folder, String name, int sizeBytes) throws IOException { + byte[] payload = new byte[sizeBytes]; + Arrays.fill(payload, (byte) 1); + Files.write(folder.resolve(name), payload); + } + + private static void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java index d35d3a52ea291..e314093d9f41c 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java @@ -28,6 +28,10 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; public class JVMCommonUtilsTest { @@ -66,4 +70,48 @@ public void getOccupiedSpaceSumsFileSizes() throws IOException { Assert.assertEquals( 2L * payload.length, JVMCommonUtils.getOccupiedSpace(dir.getAbsolutePath())); } + + @Test + public void getOccupiedSpaceIgnoresConcurrentlyDeletedEntries() throws Exception { + File snapshotRoot = tempFolder.newFolder("snapshot"); + File subDir = new File(snapshotRoot, "tod_sod0-71"); + Assert.assertTrue(subDir.mkdirs()); + byte[] payload = "snapshot-data".getBytes(StandardCharsets.UTF_8); + Files.write(new File(subDir, "a.bin").toPath(), payload); + Files.write(new File(snapshotRoot, "other.bin").toPath(), payload); + + AtomicBoolean stop = new AtomicBoolean(false); + Thread deleter = + new Thread( + () -> { + while (!stop.get()) { + try { + deleteRecursively(subDir.toPath()); + Files.createDirectories(subDir.toPath()); + Files.write(new File(subDir, "temp.bin").toPath(), payload); + } catch (IOException ignored) { + // Concurrent create/delete is expected in this test. + } + } + }); + deleter.start(); + + try { + for (int i = 0; i < 200; i++) { + JVMCommonUtils.getOccupiedSpace(snapshotRoot.getAbsolutePath()); + } + } finally { + stop.set(true); + deleter.join(5000); + } + } + + private static void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } }