Skip to content

Commit b35a575

Browse files
authored
Fix deletion visibility in region migration IT (#18065)
1 parent 186ce36 commit b35a575

3 files changed

Lines changed: 203 additions & 80 deletions

File tree

integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java

Lines changed: 18 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,18 @@
3737
import java.sql.Connection;
3838
import java.sql.ResultSet;
3939
import java.sql.Statement;
40-
import java.util.HashSet;
4140
import java.util.Map;
4241
import java.util.Set;
4342
import java.util.concurrent.TimeUnit;
4443

4544
import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getAllDataNodes;
4645
import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMapWithLeader;
46+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.MULTI_DATA_DIRS;
47+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitModsVisibleOnReplicas;
48+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitRegionReplicas;
49+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileResourceVisibleOnReplicas;
50+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileVisibleOnReplicas;
51+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.getReplicaDataNodeIds;
4752

4853
/**
4954
* Tree-model coverage for IoTConsensus region migration over multiple data dirs: a deletion (mods)
@@ -56,9 +61,6 @@
5661
@Category({ClusterIT.class})
5762
public class IoTDBRegionMigrateWithDeletionMultiDataDirIT {
5863

59-
private static final String MULTI_DATA_DIRS =
60-
"data/datanode/data/disk0,data/datanode/data/disk1,data/datanode/data/disk2";
61-
6264
@Before
6365
public void setUp() throws Exception {
6466
EnvFactory.getEnv()
@@ -83,13 +85,21 @@ public void testRegionMigratePreservesDeletionWithMultiDataDirs() throws Excepti
8385
statement.execute(
8486
"INSERT INTO root.db.d1(timestamp, s1) VALUES (100, 100), (200, 200), (300, 300)");
8587
statement.execute("FLUSH");
86-
statement.execute("DELETE FROM root.db.d1.s1 WHERE time <= 200");
87-
statement.execute("FLUSH");
8888

8989
Map<Integer, Pair<Integer, Set<Integer>>> dataRegionMapWithLeader =
9090
getDataRegionMapWithLeader(statement);
9191
int dataRegionIdForTest =
9292
dataRegionMapWithLeader.keySet().stream().max(Integer::compareTo).orElseThrow();
93+
Set<Integer> initialReplicaDataNodeIds =
94+
getReplicaDataNodeIds(statement, dataRegionIdForTest);
95+
96+
awaitTsFileVisibleOnReplicas("root.db", dataRegionIdForTest, initialReplicaDataNodeIds);
97+
awaitTsFileResourceVisibleOnReplicas(
98+
statement, "root.db", dataRegionIdForTest, initialReplicaDataNodeIds);
99+
100+
statement.execute("DELETE FROM root.db.d1.s1 WHERE time <= 200");
101+
statement.execute("FLUSH");
102+
awaitModsVisibleOnReplicas("root.db", dataRegionIdForTest, initialReplicaDataNodeIds);
93103
assertDeletionVisibleOnAllReplicas(dataRegionIdForTest, 1);
94104

95105
Pair<Integer, Set<Integer>> leaderAndNodes = dataRegionMapWithLeader.get(dataRegionIdForTest);
@@ -107,25 +117,8 @@ public void testRegionMigratePreservesDeletionWithMultiDataDirs() throws Excepti
107117
String.format(
108118
"migrate region %d from %d to %d", dataRegionIdForTest, leaderId, destDataNodeId));
109119

110-
final int finalDestDataNodeId = destDataNodeId;
111-
Awaitility.await()
112-
.atMost(10, TimeUnit.MINUTES)
113-
.pollDelay(1, TimeUnit.SECONDS)
114-
.pollInterval(2, TimeUnit.SECONDS)
115-
.untilAsserted(
116-
() -> {
117-
try (ResultSet showRegions = statement.executeQuery("SHOW REGIONS")) {
118-
boolean migrated = false;
119-
while (showRegions.next()) {
120-
if (showRegions.getInt("RegionId") == dataRegionIdForTest
121-
&& showRegions.getInt("DataNodeId") == finalDestDataNodeId) {
122-
migrated = true;
123-
break;
124-
}
125-
}
126-
Assert.assertTrue(migrated);
127-
}
128-
});
120+
awaitRegionReplicas(statement, dataRegionIdForTest, Set.of(followerId, destDataNodeId));
121+
awaitModsVisibleOnReplicas("root.db", dataRegionIdForTest, Set.of(destDataNodeId));
129122

130123
assertDeletionVisibleOnAllReplicas(dataRegionIdForTest, 1);
131124
}
@@ -164,19 +157,4 @@ private void assertDeletionVisibleOnReplica(DataNodeWrapper dataNodeWrapper, int
164157
}
165158
}
166159
}
167-
168-
private Set<Integer> getReplicaDataNodeIds(Statement statement, int dataRegionId)
169-
throws Exception {
170-
Set<Integer> replicaDataNodeIds = new HashSet<>();
171-
try (ResultSet showRegions = statement.executeQuery("SHOW REGIONS")) {
172-
while (showRegions.next()) {
173-
if ("DataRegion".equals(showRegions.getString("Type"))
174-
&& showRegions.getInt("RegionId") == dataRegionId) {
175-
replicaDataNodeIds.add(showRegions.getInt("DataNodeId"));
176-
}
177-
}
178-
}
179-
Assert.assertFalse(replicaDataNodeIds.isEmpty());
180-
return replicaDataNodeIds;
181-
}
182160
}

integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/daily/iotv1/IoTDBRegionMigrateWithDeletionMultiDataDirTableIT.java

Lines changed: 18 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,18 @@
3939
import java.sql.Connection;
4040
import java.sql.ResultSet;
4141
import java.sql.Statement;
42-
import java.util.HashSet;
4342
import java.util.Map;
4443
import java.util.Set;
4544
import java.util.concurrent.TimeUnit;
4645

4746
import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getAllDataNodes;
4847
import static org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionOperationReliabilityITFramework.getDataRegionMapWithLeader;
48+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.MULTI_DATA_DIRS;
49+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitModsVisibleOnReplicas;
50+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitRegionReplicas;
51+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileResourceVisibleOnReplicas;
52+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.awaitTsFileVisibleOnReplicas;
53+
import static org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1.RegionMigrateFileAssertions.getReplicaDataNodeIds;
4954

5055
/**
5156
* Table-model twin of {@link IoTDBRegionMigrateWithDeletionMultiDataDirIT}: a deletion (mods) must
@@ -56,9 +61,6 @@
5661
@Category({TableClusterIT.class})
5762
public class IoTDBRegionMigrateWithDeletionMultiDataDirTableIT {
5863

59-
private static final String MULTI_DATA_DIRS =
60-
"data/datanode/data/disk0,data/datanode/data/disk1,data/datanode/data/disk2";
61-
6264
@Before
6365
public void setUp() throws Exception {
6466
EnvFactory.getEnv()
@@ -84,13 +86,21 @@ public void testRegionMigratePreservesDeletionWithMultiDataDirs() throws Excepti
8486
statement.execute("CREATE TABLE t1 (s1 INT64 FIELD)");
8587
statement.execute("INSERT INTO t1 (time, s1) VALUES (100, 100), (200, 200), (300, 300)");
8688
statement.execute("FLUSH");
87-
statement.execute("DELETE FROM t1 WHERE time <= 200");
88-
statement.execute("FLUSH");
8989

9090
Map<Integer, Pair<Integer, Set<Integer>>> dataRegionMapWithLeader =
9191
getDataRegionMapWithLeader(statement);
9292
int dataRegionIdForTest =
9393
dataRegionMapWithLeader.keySet().stream().max(Integer::compareTo).orElseThrow();
94+
Set<Integer> initialReplicaDataNodeIds =
95+
getReplicaDataNodeIds(statement, dataRegionIdForTest);
96+
97+
awaitTsFileVisibleOnReplicas("test", dataRegionIdForTest, initialReplicaDataNodeIds);
98+
awaitTsFileResourceVisibleOnReplicas(
99+
statement, "test", dataRegionIdForTest, initialReplicaDataNodeIds);
100+
101+
statement.execute("DELETE FROM t1 WHERE time <= 200");
102+
statement.execute("FLUSH");
103+
awaitModsVisibleOnReplicas("test", dataRegionIdForTest, initialReplicaDataNodeIds);
94104
assertDeletionVisibleOnAllReplicas(statement, dataRegionIdForTest, 1);
95105

96106
Pair<Integer, Set<Integer>> leaderAndNodes = dataRegionMapWithLeader.get(dataRegionIdForTest);
@@ -108,25 +118,8 @@ public void testRegionMigratePreservesDeletionWithMultiDataDirs() throws Excepti
108118
String.format(
109119
"migrate region %d from %d to %d", dataRegionIdForTest, leaderId, destDataNodeId));
110120

111-
final int finalDestDataNodeId = destDataNodeId;
112-
Awaitility.await()
113-
.atMost(10, TimeUnit.MINUTES)
114-
.pollDelay(1, TimeUnit.SECONDS)
115-
.pollInterval(2, TimeUnit.SECONDS)
116-
.untilAsserted(
117-
() -> {
118-
try (ResultSet showRegions = statement.executeQuery("SHOW REGIONS")) {
119-
boolean migrated = false;
120-
while (showRegions.next()) {
121-
if (showRegions.getInt("RegionId") == dataRegionIdForTest
122-
&& showRegions.getInt("DataNodeId") == finalDestDataNodeId) {
123-
migrated = true;
124-
break;
125-
}
126-
}
127-
Assert.assertTrue(migrated);
128-
}
129-
});
121+
awaitRegionReplicas(statement, dataRegionIdForTest, Set.of(followerId, destDataNodeId));
122+
awaitModsVisibleOnReplicas("test", dataRegionIdForTest, Set.of(destDataNodeId));
130123

131124
assertDeletionVisibleOnAllReplicas(statement, dataRegionIdForTest, 1);
132125
}
@@ -167,19 +160,4 @@ private void assertDeletionVisibleOnReplica(DataNodeWrapper dataNodeWrapper, int
167160
}
168161
}
169162
}
170-
171-
private Set<Integer> getReplicaDataNodeIds(Statement statement, int dataRegionId)
172-
throws Exception {
173-
Set<Integer> replicaDataNodeIds = new HashSet<>();
174-
try (ResultSet showRegions = statement.executeQuery("SHOW REGIONS")) {
175-
while (showRegions.next()) {
176-
if ("DataRegion".equals(showRegions.getString("Type"))
177-
&& showRegions.getInt("RegionId") == dataRegionId) {
178-
replicaDataNodeIds.add(showRegions.getInt("DataNodeId"));
179-
}
180-
}
181-
}
182-
Assert.assertFalse(replicaDataNodeIds.isEmpty());
183-
return replicaDataNodeIds;
184-
}
185163
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.confignode.it.regionmigration.pass.daily.iotv1;
21+
22+
import org.apache.iotdb.it.env.EnvFactory;
23+
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
24+
25+
import org.awaitility.Awaitility;
26+
import org.junit.Assert;
27+
28+
import java.io.File;
29+
import java.sql.ResultSet;
30+
import java.sql.Statement;
31+
import java.util.HashSet;
32+
import java.util.Set;
33+
import java.util.concurrent.TimeUnit;
34+
35+
final class RegionMigrateFileAssertions {
36+
37+
static final String MULTI_DATA_DIRS =
38+
"data/datanode/data/disk0,data/datanode/data/disk1,data/datanode/data/disk2";
39+
40+
private static final String SEQUENCE_FOLDER = "sequence";
41+
private static final String TSFILE_SUFFIX = ".tsfile";
42+
private static final String TSFILE_RESOURCE_SUFFIX = ".tsfile.resource";
43+
private static final String MODS_SUFFIX = ".mods2";
44+
45+
private RegionMigrateFileAssertions() {}
46+
47+
static void awaitTsFileVisibleOnReplicas(
48+
String database, int dataRegionId, Set<Integer> dataNodeIds) {
49+
awaitFileVisibleOnReplicas(database, dataRegionId, dataNodeIds, TSFILE_SUFFIX);
50+
}
51+
52+
static void awaitTsFileResourceVisibleOnReplicas(
53+
String database, int dataRegionId, Set<Integer> dataNodeIds) {
54+
awaitFileVisibleOnReplicas(database, dataRegionId, dataNodeIds, TSFILE_RESOURCE_SUFFIX);
55+
}
56+
57+
static void awaitTsFileResourceVisibleOnReplicas(
58+
Statement statement, String database, int dataRegionId, Set<Integer> dataNodeIds) {
59+
awaitFileVisibleOnReplicas(
60+
statement, database, dataRegionId, dataNodeIds, TSFILE_RESOURCE_SUFFIX);
61+
}
62+
63+
static void awaitModsVisibleOnReplicas(
64+
String database, int dataRegionId, Set<Integer> dataNodeIds) {
65+
awaitFileVisibleOnReplicas(database, dataRegionId, dataNodeIds, MODS_SUFFIX);
66+
}
67+
68+
static void awaitRegionReplicas(
69+
Statement statement, int dataRegionId, Set<Integer> expectedReplicaDataNodeIds) {
70+
Awaitility.await()
71+
.atMost(10, TimeUnit.MINUTES)
72+
.pollDelay(1, TimeUnit.SECONDS)
73+
.pollInterval(2, TimeUnit.SECONDS)
74+
.untilAsserted(
75+
() ->
76+
Assert.assertEquals(
77+
expectedReplicaDataNodeIds, getReplicaDataNodeIds(statement, dataRegionId)));
78+
}
79+
80+
static Set<Integer> getReplicaDataNodeIds(Statement statement, int dataRegionId)
81+
throws Exception {
82+
Set<Integer> replicaDataNodeIds = new HashSet<>();
83+
try (ResultSet showRegions = statement.executeQuery("SHOW REGIONS")) {
84+
while (showRegions.next()) {
85+
if ("DataRegion".equals(showRegions.getString("Type"))
86+
&& showRegions.getInt("RegionId") == dataRegionId) {
87+
replicaDataNodeIds.add(showRegions.getInt("DataNodeId"));
88+
}
89+
}
90+
}
91+
Assert.assertFalse(replicaDataNodeIds.isEmpty());
92+
return replicaDataNodeIds;
93+
}
94+
95+
private static void awaitFileVisibleOnReplicas(
96+
String database, int dataRegionId, Set<Integer> dataNodeIds, String suffix) {
97+
awaitFileVisibleOnReplicas(null, database, dataRegionId, dataNodeIds, suffix);
98+
}
99+
100+
private static void awaitFileVisibleOnReplicas(
101+
Statement flushStatement,
102+
String database,
103+
int dataRegionId,
104+
Set<Integer> dataNodeIds,
105+
String suffix) {
106+
for (int dataNodeId : dataNodeIds) {
107+
DataNodeWrapper dataNodeWrapper =
108+
EnvFactory.getEnv().dataNodeIdToWrapper(dataNodeId).orElseThrow();
109+
Awaitility.await()
110+
.atMost(2, TimeUnit.MINUTES)
111+
.pollDelay(500, TimeUnit.MILLISECONDS)
112+
.pollInterval(1, TimeUnit.SECONDS)
113+
.untilAsserted(
114+
() -> {
115+
if (flushStatement != null) {
116+
flushStatement.execute("FLUSH");
117+
}
118+
Assert.assertTrue(
119+
String.format(
120+
"Expected file with suffix %s for database %s region %d on DataNode %d",
121+
suffix, database, dataRegionId, dataNodeId),
122+
containsSequenceFileWithSuffix(
123+
dataNodeWrapper, database, dataRegionId, suffix));
124+
});
125+
}
126+
}
127+
128+
private static boolean containsSequenceFileWithSuffix(
129+
DataNodeWrapper dataNodeWrapper, String database, int dataRegionId, String suffix) {
130+
for (String dataDir : MULTI_DATA_DIRS.split(",")) {
131+
File regionDir =
132+
new File(
133+
dataNodeWrapper.getNodePath(),
134+
dataDir
135+
+ File.separator
136+
+ SEQUENCE_FOLDER
137+
+ File.separator
138+
+ database
139+
+ File.separator
140+
+ dataRegionId);
141+
if (containsFileWithSuffix(regionDir, suffix)) {
142+
return true;
143+
}
144+
}
145+
return false;
146+
}
147+
148+
private static boolean containsFileWithSuffix(File file, String suffix) {
149+
if (!file.exists()) {
150+
return false;
151+
}
152+
if (file.isFile()) {
153+
// IoTConsensus followers can create an open zero-byte TsFile before a later FLUSH closes it.
154+
return file.getName().endsWith(suffix) && (TSFILE_SUFFIX.equals(suffix) || file.length() > 0);
155+
}
156+
File[] children = file.listFiles();
157+
if (children == null) {
158+
return false;
159+
}
160+
for (File child : children) {
161+
if (containsFileWithSuffix(child, suffix)) {
162+
return true;
163+
}
164+
}
165+
return false;
166+
}
167+
}

0 commit comments

Comments
 (0)