From e94a3ccf8b135dc929350b6ce689b21c7bda55ab Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Mon, 11 Dec 2023 11:46:24 -0500 Subject: [PATCH 01/11] Functionality to run ScanConsistencyIT standalone Closes #3646. Refactored ScanConsistencyIT to be able to be run standalone. Can now be run as a test or from main. --- .../accumulo/test/ScanConsistencyIT.java | 183 ++++++++++++------ 1 file changed, 129 insertions(+), 54 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index ba5873bed71..c8554812006 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -79,12 +79,58 @@ public class ScanConsistencyIT extends AccumuloClusterHarness { private static final Logger log = LoggerFactory.getLogger(ScanConsistencyIT.class); + private static boolean inTestingContext; + + public static void main(String[] args) { + /** + * @formatter:off + * Note: In order to run main, + * 1) Build the project + * 2) Copy the accumulo test jar (in /test/target/) to your accumulo installation's + * lib directory* + * Now, this can be run with + * "accumulo org.apache.accumulo.test.ScanConsistencyIT " + * : An accumulo client properties file + * : tmpDir field for the TestContext object + *
: The name of the table to be created + * *Ensure the test jar is in lib before the tablet servers start. Restart tablet + * servers if necessary. + * @formatter:on + */ + if (args.length == 3) { + inTestingContext = false; + final String propsFile = args[0]; + final String tmpDir = args[1]; + final String table = args[2]; + + try { + AccumuloClient client = Accumulo.newClient().from(propsFile).build(); + FileSystem fileSystem = FileSystem.get(new Configuration()); + runTest(client, fileSystem, tmpDir, table); + } catch (Exception e) { + log.error(e.toString()); + } + } else { + log.error("Invalid arguments. Use: " + + "accumulo org.apache.accumulo.test.ScanConsistencyIT
"); + } + } @SuppressFBWarnings(value = {"PREDICTABLE_RANDOM", "DMI_RANDOM_USED_ONLY_ONCE"}, justification = "predictable random is ok for testing") @Test public void testConcurrentScanConsistency() throws Exception { - final String table = this.getUniqueNames(1)[0]; + inTestingContext = true; + try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { + FileSystem fileSystem = getCluster().getFileSystem(); + final String tmpDir = getCluster().getTemporaryPath().toString(); + final String table = getUniqueNames(1)[0]; + runTest(client, fileSystem, tmpDir, table); + } + } + + private static void runTest(AccumuloClient client, FileSystem fileSystem, String tmpDir, + String table) throws Exception { /** * Tips for debugging this test when it sees a row that should not exist or does not see a row @@ -104,69 +150,97 @@ public void testConcurrentScanConsistency() throws Exception { // getClusterControl().stopAllServers(ServerType.GARBAGE_COLLECTOR); var executor = Executors.newCachedThreadPool(); - try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { - client.tableOperations().create(table); + client.tableOperations().create(table); - TestContext testContext = new TestContext(client, table, getCluster().getFileSystem(), - getCluster().getTemporaryPath().toString()); + TestContext testContext = new TestContext(client, table, fileSystem, tmpDir); - List> writeTasks = new ArrayList<>(); - List> scanTasks = new ArrayList<>(); + List> writeTasks = new ArrayList<>(); + List> scanTasks = new ArrayList<>(); - Random random = new Random(); + Random random = new Random(); - int numWriteTask = random.nextInt(10) + 1; - int numsScanTask = random.nextInt(10) + 1; + int numWriteTask = random.nextInt(10) + 1; + int numsScanTask = random.nextInt(10) + 1; - for (int i = 0; i < numWriteTask; i++) { - writeTasks.add(executor.submit(new WriteTask(testContext))); - } + for (int i = 0; i < numWriteTask; i++) { + writeTasks.add(executor.submit(new WriteTask(testContext))); + } - for (int i = 0; i < numsScanTask; i++) { - scanTasks.add(executor.submit(new ScanTask(testContext))); - } + for (int i = 0; i < numsScanTask; i++) { + scanTasks.add(executor.submit(new ScanTask(testContext))); + } - var tableOpsTask = executor.submit(new TableOpsTask(testContext)); + var tableOpsTask = executor.submit(new TableOpsTask(testContext)); - // let the concurrent mayhem run for a bit - Thread.sleep(60000); + // let the concurrent mayhem run for a bit + Thread.sleep(60000); - // let the threads know to exit - testContext.keepRunning.set(false); + // let the threads know to exit + testContext.keepRunning.set(false); - for (Future writeTask : writeTasks) { - var stats = writeTask.get(); - log.debug(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", - stats.written, stats.bulkImported, stats.deleted, stats.bulkDeleted)); - assertTrue(stats.written + stats.bulkImported > 0); - assertTrue(stats.deleted + stats.bulkDeleted > 0); - } + for (Future writeTask : writeTasks) { + var stats = writeTask.get(); + log.debug(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", + stats.written, stats.bulkImported, stats.deleted, stats.bulkDeleted)); + checkTrue(stats.written + stats.bulkImported > 0); + checkTrue(stats.deleted + stats.bulkDeleted > 0); + } - for (Future scanTask : scanTasks) { - var stats = scanTask.get(); - log.debug(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); - assertTrue(stats.verified > 0); - // These scans were running concurrently with writes, so a scan will see more data than what - // was written before the scan started. - assertTrue(stats.scanned > stats.verified); - } + for (Future scanTask : scanTasks) { + var stats = scanTask.get(); + log.debug(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); + checkTrue(stats.verified > 0); + // These scans were running concurrently with writes, so a scan will see more data than what + // was written before the scan started. + checkTrue(stats.scanned > stats.verified); + } + + log.debug(tableOpsTask.get()); + + var stats1 = scanData(testContext, random, new Range(), false); + var stats2 = scanData(testContext, random, new Range(), true); + var stats3 = batchScanData(testContext, new Range()); + log.debug( + String.format("Final scan, scanned:%,d verified:%,d", stats1.scanned, stats1.verified)); + checkTrue(stats1.verified > 0); + // Should see all expected data now that there are no concurrent writes happening + checkEquals(stats1.scanned, stats1.verified); + checkEquals(stats2.scanned, stats1.scanned); + checkEquals(stats2.verified, stats1.verified); + checkEquals(stats3.scanned, stats1.scanned); + checkEquals(stats3.verified, stats1.verified); + + executor.shutdownNow(); + } - log.debug(tableOpsTask.get()); - - var stats1 = scanData(testContext, random, new Range(), false); - var stats2 = scanData(testContext, random, new Range(), true); - var stats3 = batchScanData(testContext, new Range()); - log.debug( - String.format("Final scan, scanned:%,d verified:%,d", stats1.scanned, stats1.verified)); - assertTrue(stats1.verified > 0); - // Should see all expected data now that there are no concurrent writes happening - assertEquals(stats1.scanned, stats1.verified); - assertEquals(stats2.scanned, stats1.scanned); - assertEquals(stats2.verified, stats1.verified); - assertEquals(stats3.scanned, stats1.scanned); - assertEquals(stats3.verified, stats1.verified); - } finally { - executor.shutdownNow(); + /** + * Checks if b is true. Uses JUnit assert if inTestingContext, otherwise checks if b is true and + * throws an exception if not. + * + * @param b The boolean checked + * @throws Exception + */ + private static void checkTrue(boolean b) throws Exception { + if (inTestingContext) { + assertTrue(b); + } else if (!b) { + throw new Exception("Failed assertion"); + } + } + + /** + * Checks if l1 and l2 are equal. Uses JUnit assert if inTestingContext, otherwise checks if l1 + * and l2 are equal and throws an exception if not. + * + * @param l1 One of the two longs to be compared + * @param l2 One of the two longs to be compared + * @throws Exception + */ + private static void checkEquals(long l1, long l2) throws Exception { + if (inTestingContext) { + assertEquals(l1, l2); + } else if (l1 != l2) { + throw new Exception("Failed assertion"); } } @@ -324,7 +398,8 @@ public void add(ScanStats stats) { } } - private static ScanStats scan(Stream> scanner, Set expected) { + private static ScanStats scan(Stream> scanner, Set expected) + throws Exception { ScanStats stats = new ScanStats(); scanner.forEach(entry -> { @@ -336,7 +411,7 @@ private static ScanStats scan(Stream> scanner, Set exp } }); - assertTrue(expected.isEmpty()); + checkTrue(expected.isEmpty()); return stats; } From 68403d57e17f918750bd02e4598249fa1a97b678 Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Mon, 11 Dec 2023 13:01:01 -0500 Subject: [PATCH 02/11] Requested changes --- .../accumulo/test/ScanConsistencyIT.java | 132 +++++++++--------- 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index c8554812006..521534cd86d 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -97,22 +97,19 @@ public static void main(String[] args) { * servers if necessary. * @formatter:on */ - if (args.length == 3) { - inTestingContext = false; - final String propsFile = args[0]; - final String tmpDir = args[1]; - final String table = args[2]; - - try { - AccumuloClient client = Accumulo.newClient().from(propsFile).build(); - FileSystem fileSystem = FileSystem.get(new Configuration()); - runTest(client, fileSystem, tmpDir, table); - } catch (Exception e) { - log.error(e.toString()); - } - } else { - log.error("Invalid arguments. Use: " - + "accumulo org.apache.accumulo.test.ScanConsistencyIT
"); + Preconditions.checkArgument(args.length == 3, "Invalid arguments. Use: " + + "accumulo org.apache.accumulo.test.ScanConsistencyIT
"); + inTestingContext = false; + final String propsFile = args[0]; + final String tmpDir = args[1]; + final String table = args[2]; + + try { + AccumuloClient client = Accumulo.newClient().from(propsFile).build(); + FileSystem fileSystem = FileSystem.get(new Configuration()); + runTest(client, fileSystem, tmpDir, table); + } catch (Exception e) { + log.error(e.toString()); } } @@ -150,67 +147,70 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String // getClusterControl().stopAllServers(ServerType.GARBAGE_COLLECTOR); var executor = Executors.newCachedThreadPool(); - client.tableOperations().create(table); + try { + client.tableOperations().create(table); - TestContext testContext = new TestContext(client, table, fileSystem, tmpDir); + TestContext testContext = new TestContext(client, table, fileSystem, tmpDir); - List> writeTasks = new ArrayList<>(); - List> scanTasks = new ArrayList<>(); + List> writeTasks = new ArrayList<>(); + List> scanTasks = new ArrayList<>(); - Random random = new Random(); - - int numWriteTask = random.nextInt(10) + 1; - int numsScanTask = random.nextInt(10) + 1; - - for (int i = 0; i < numWriteTask; i++) { - writeTasks.add(executor.submit(new WriteTask(testContext))); - } + Random random = new Random(); - for (int i = 0; i < numsScanTask; i++) { - scanTasks.add(executor.submit(new ScanTask(testContext))); - } + int numWriteTask = random.nextInt(10) + 1; + int numsScanTask = random.nextInt(10) + 1; - var tableOpsTask = executor.submit(new TableOpsTask(testContext)); + for (int i = 0; i < numWriteTask; i++) { + writeTasks.add(executor.submit(new WriteTask(testContext))); + } - // let the concurrent mayhem run for a bit - Thread.sleep(60000); + for (int i = 0; i < numsScanTask; i++) { + scanTasks.add(executor.submit(new ScanTask(testContext))); + } - // let the threads know to exit - testContext.keepRunning.set(false); + var tableOpsTask = executor.submit(new TableOpsTask(testContext)); - for (Future writeTask : writeTasks) { - var stats = writeTask.get(); - log.debug(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", - stats.written, stats.bulkImported, stats.deleted, stats.bulkDeleted)); - checkTrue(stats.written + stats.bulkImported > 0); - checkTrue(stats.deleted + stats.bulkDeleted > 0); - } + // let the concurrent mayhem run for a bit + Thread.sleep(60000); - for (Future scanTask : scanTasks) { - var stats = scanTask.get(); - log.debug(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); - checkTrue(stats.verified > 0); - // These scans were running concurrently with writes, so a scan will see more data than what - // was written before the scan started. - checkTrue(stats.scanned > stats.verified); - } + // let the threads know to exit + testContext.keepRunning.set(false); - log.debug(tableOpsTask.get()); + for (Future writeTask : writeTasks) { + var stats = writeTask.get(); + log.debug(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", + stats.written, stats.bulkImported, stats.deleted, stats.bulkDeleted)); + checkTrue(stats.written + stats.bulkImported > 0); + checkTrue(stats.deleted + stats.bulkDeleted > 0); + } - var stats1 = scanData(testContext, random, new Range(), false); - var stats2 = scanData(testContext, random, new Range(), true); - var stats3 = batchScanData(testContext, new Range()); - log.debug( - String.format("Final scan, scanned:%,d verified:%,d", stats1.scanned, stats1.verified)); - checkTrue(stats1.verified > 0); - // Should see all expected data now that there are no concurrent writes happening - checkEquals(stats1.scanned, stats1.verified); - checkEquals(stats2.scanned, stats1.scanned); - checkEquals(stats2.verified, stats1.verified); - checkEquals(stats3.scanned, stats1.scanned); - checkEquals(stats3.verified, stats1.verified); + for (Future scanTask : scanTasks) { + var stats = scanTask.get(); + log.debug(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); + checkTrue(stats.verified > 0); + // These scans were running concurrently with writes, so a scan will see more data than what + // was written before the scan started. + checkTrue(stats.scanned > stats.verified); + } - executor.shutdownNow(); + log.debug(tableOpsTask.get()); + + var stats1 = scanData(testContext, random, new Range(), false); + var stats2 = scanData(testContext, random, new Range(), true); + var stats3 = batchScanData(testContext, new Range()); + log.debug( + String.format("Final scan, scanned:%,d verified:%,d", stats1.scanned, stats1.verified)); + checkTrue(stats1.verified > 0); + // Should see all expected data now that there are no concurrent writes happening + checkEquals(stats1.scanned, stats1.verified); + checkEquals(stats2.scanned, stats1.scanned); + checkEquals(stats2.verified, stats1.verified); + checkEquals(stats3.scanned, stats1.scanned); + checkEquals(stats3.verified, stats1.verified); + } finally { + executor.shutdownNow(); + client.tableOperations().delete(table); + } } /** @@ -218,7 +218,6 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String * throws an exception if not. * * @param b The boolean checked - * @throws Exception */ private static void checkTrue(boolean b) throws Exception { if (inTestingContext) { @@ -234,7 +233,6 @@ private static void checkTrue(boolean b) throws Exception { * * @param l1 One of the two longs to be compared * @param l2 One of the two longs to be compared - * @throws Exception */ private static void checkEquals(long l1, long l2) throws Exception { if (inTestingContext) { From 471c4afbc155ce47cf564587ba91a3187f0eab40 Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Mon, 11 Dec 2023 13:20:00 -0500 Subject: [PATCH 03/11] More requested changes --- .../accumulo/test/ScanConsistencyIT.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index 521534cd86d..67950e492a7 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -104,8 +104,7 @@ public static void main(String[] args) { final String tmpDir = args[1]; final String table = args[2]; - try { - AccumuloClient client = Accumulo.newClient().from(propsFile).build(); + try (AccumuloClient client = Accumulo.newClient().from(propsFile).build()) { FileSystem fileSystem = FileSystem.get(new Configuration()); runTest(client, fileSystem, tmpDir, table); } catch (Exception e) { @@ -178,7 +177,7 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String for (Future writeTask : writeTasks) { var stats = writeTask.get(); - log.debug(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", + logMessage(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", stats.written, stats.bulkImported, stats.deleted, stats.bulkDeleted)); checkTrue(stats.written + stats.bulkImported > 0); checkTrue(stats.deleted + stats.bulkDeleted > 0); @@ -186,19 +185,19 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String for (Future scanTask : scanTasks) { var stats = scanTask.get(); - log.debug(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); + logMessage(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); checkTrue(stats.verified > 0); // These scans were running concurrently with writes, so a scan will see more data than what // was written before the scan started. checkTrue(stats.scanned > stats.verified); } - log.debug(tableOpsTask.get()); + logMessage(tableOpsTask.get()); var stats1 = scanData(testContext, random, new Range(), false); var stats2 = scanData(testContext, random, new Range(), true); var stats3 = batchScanData(testContext, new Range()); - log.debug( + logMessage( String.format("Final scan, scanned:%,d verified:%,d", stats1.scanned, stats1.verified)); checkTrue(stats1.verified > 0); // Should see all expected data now that there are no concurrent writes happening @@ -242,6 +241,19 @@ private static void checkEquals(long l1, long l2) throws Exception { } } + /** + * Logs msg at debug level if inTestingContext, info level otherwise + * + * @param msg message to be logged + */ + private static void logMessage(String msg) { + if (inTestingContext) { + log.debug(msg); + } else { + log.info(msg); + } + } + /** * Tracks what data has been written and deleted in an Accumulo table. */ From bf6b87ca6d8c6bbe28aa5f8505bd408776fe69d3 Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Mon, 11 Dec 2023 14:46:27 -0500 Subject: [PATCH 04/11] fixed table deletion, use log.info, accept sleep time --- .../accumulo/test/ScanConsistencyIT.java | 41 ++++++++----------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index 67950e492a7..c69bdf9a24c 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -89,24 +89,26 @@ public static void main(String[] args) { * 2) Copy the accumulo test jar (in /test/target/) to your accumulo installation's * lib directory* * Now, this can be run with - * "accumulo org.apache.accumulo.test.ScanConsistencyIT
" + * "accumulo org.apache.accumulo.test.ScanConsistencyIT
" * : An accumulo client properties file * : tmpDir field for the TestContext object *
: The name of the table to be created + * : The time to sleep (ms) after submitting the various concurrent tasks * *Ensure the test jar is in lib before the tablet servers start. Restart tablet * servers if necessary. * @formatter:on */ - Preconditions.checkArgument(args.length == 3, "Invalid arguments. Use: " - + "accumulo org.apache.accumulo.test.ScanConsistencyIT
"); + Preconditions.checkArgument(args.length == 4, "Invalid arguments. Use: " + + "accumulo org.apache.accumulo.test.ScanConsistencyIT
"); inTestingContext = false; final String propsFile = args[0]; final String tmpDir = args[1]; final String table = args[2]; + final long sleepTime = Long.parseLong(args[3]); try (AccumuloClient client = Accumulo.newClient().from(propsFile).build()) { FileSystem fileSystem = FileSystem.get(new Configuration()); - runTest(client, fileSystem, tmpDir, table); + runTest(client, fileSystem, tmpDir, table, sleepTime); } catch (Exception e) { log.error(e.toString()); } @@ -121,12 +123,13 @@ public void testConcurrentScanConsistency() throws Exception { FileSystem fileSystem = getCluster().getFileSystem(); final String tmpDir = getCluster().getTemporaryPath().toString(); final String table = getUniqueNames(1)[0]; - runTest(client, fileSystem, tmpDir, table); + final long sleepTime = 60000; + runTest(client, fileSystem, tmpDir, table, sleepTime); } } private static void runTest(AccumuloClient client, FileSystem fileSystem, String tmpDir, - String table) throws Exception { + String table, long sleepTime) throws Exception { /** * Tips for debugging this test when it sees a row that should not exist or does not see a row @@ -170,14 +173,14 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String var tableOpsTask = executor.submit(new TableOpsTask(testContext)); // let the concurrent mayhem run for a bit - Thread.sleep(60000); + Thread.sleep(sleepTime); // let the threads know to exit testContext.keepRunning.set(false); for (Future writeTask : writeTasks) { var stats = writeTask.get(); - logMessage(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", + log.info(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", stats.written, stats.bulkImported, stats.deleted, stats.bulkDeleted)); checkTrue(stats.written + stats.bulkImported > 0); checkTrue(stats.deleted + stats.bulkDeleted > 0); @@ -185,19 +188,19 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String for (Future scanTask : scanTasks) { var stats = scanTask.get(); - logMessage(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); + log.info(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); checkTrue(stats.verified > 0); // These scans were running concurrently with writes, so a scan will see more data than what // was written before the scan started. checkTrue(stats.scanned > stats.verified); } - logMessage(tableOpsTask.get()); + log.info(tableOpsTask.get()); var stats1 = scanData(testContext, random, new Range(), false); var stats2 = scanData(testContext, random, new Range(), true); var stats3 = batchScanData(testContext, new Range()); - logMessage( + log.info( String.format("Final scan, scanned:%,d verified:%,d", stats1.scanned, stats1.verified)); checkTrue(stats1.verified > 0); // Should see all expected data now that there are no concurrent writes happening @@ -206,9 +209,10 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String checkEquals(stats2.verified, stats1.verified); checkEquals(stats3.scanned, stats1.scanned); checkEquals(stats3.verified, stats1.verified); + + client.tableOperations().delete(table); } finally { executor.shutdownNow(); - client.tableOperations().delete(table); } } @@ -241,19 +245,6 @@ private static void checkEquals(long l1, long l2) throws Exception { } } - /** - * Logs msg at debug level if inTestingContext, info level otherwise - * - * @param msg message to be logged - */ - private static void logMessage(String msg) { - if (inTestingContext) { - log.debug(msg); - } else { - log.info(msg); - } - } - /** * Tracks what data has been written and deleted in an Accumulo table. */ From e245a4501b9172a54a06056023c7a0ffd0a4cfba Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Tue, 12 Dec 2023 09:14:29 -0500 Subject: [PATCH 05/11] Removed assertion wrapper, added note about JUnit deps --- .../accumulo/test/ScanConsistencyIT.java | 65 ++++++------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index c69bdf9a24c..0d125bc7ba0 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -79,28 +79,31 @@ public class ScanConsistencyIT extends AccumuloClusterHarness { private static final Logger log = LoggerFactory.getLogger(ScanConsistencyIT.class); - private static boolean inTestingContext; public static void main(String[] args) { /** * @formatter:off * Note: In order to run main, * 1) Build the project - * 2) Copy the accumulo test jar (in /test/target/) to your accumulo installation's + * 2) Copy the accumulo test jar (in /test/target/) into your accumulo installation's * lib directory* + * 3) Copy the JUnit dependencies into your accumulo installation's lib directory: + * "mvn dependency:copy-dependencies -DincludeGroupIds="org.junit.jupiter" + * cp test/target/dependency/junit-jupiter-* $ACCUMULO_HOME/lib/" + * + * *Ensure the test jar is in lib before the tablet servers start. Restart tablet + * servers if necessary. + * * Now, this can be run with * "accumulo org.apache.accumulo.test.ScanConsistencyIT
" * : An accumulo client properties file * : tmpDir field for the TestContext object *
: The name of the table to be created * : The time to sleep (ms) after submitting the various concurrent tasks - * *Ensure the test jar is in lib before the tablet servers start. Restart tablet - * servers if necessary. * @formatter:on */ Preconditions.checkArgument(args.length == 4, "Invalid arguments. Use: " + "accumulo org.apache.accumulo.test.ScanConsistencyIT
"); - inTestingContext = false; final String propsFile = args[0]; final String tmpDir = args[1]; final String table = args[2]; @@ -118,7 +121,6 @@ public static void main(String[] args) { justification = "predictable random is ok for testing") @Test public void testConcurrentScanConsistency() throws Exception { - inTestingContext = true; try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { FileSystem fileSystem = getCluster().getFileSystem(); final String tmpDir = getCluster().getTemporaryPath().toString(); @@ -182,17 +184,17 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String var stats = writeTask.get(); log.info(String.format("Wrote:%,d Bulk imported:%,d Deleted:%,d Bulk deleted:%,d", stats.written, stats.bulkImported, stats.deleted, stats.bulkDeleted)); - checkTrue(stats.written + stats.bulkImported > 0); - checkTrue(stats.deleted + stats.bulkDeleted > 0); + assertTrue(stats.written + stats.bulkImported > 0); + assertTrue(stats.deleted + stats.bulkDeleted > 0); } for (Future scanTask : scanTasks) { var stats = scanTask.get(); log.info(String.format("Scanned:%,d verified:%,d", stats.scanned, stats.verified)); - checkTrue(stats.verified > 0); + assertTrue(stats.verified > 0); // These scans were running concurrently with writes, so a scan will see more data than what // was written before the scan started. - checkTrue(stats.scanned > stats.verified); + assertTrue(stats.scanned > stats.verified); } log.info(tableOpsTask.get()); @@ -202,13 +204,13 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String var stats3 = batchScanData(testContext, new Range()); log.info( String.format("Final scan, scanned:%,d verified:%,d", stats1.scanned, stats1.verified)); - checkTrue(stats1.verified > 0); + assertTrue(stats1.verified > 0); // Should see all expected data now that there are no concurrent writes happening - checkEquals(stats1.scanned, stats1.verified); - checkEquals(stats2.scanned, stats1.scanned); - checkEquals(stats2.verified, stats1.verified); - checkEquals(stats3.scanned, stats1.scanned); - checkEquals(stats3.verified, stats1.verified); + assertEquals(stats1.scanned, stats1.verified); + assertEquals(stats2.scanned, stats1.scanned); + assertEquals(stats2.verified, stats1.verified); + assertEquals(stats3.scanned, stats1.scanned); + assertEquals(stats3.verified, stats1.verified); client.tableOperations().delete(table); } finally { @@ -216,35 +218,6 @@ private static void runTest(AccumuloClient client, FileSystem fileSystem, String } } - /** - * Checks if b is true. Uses JUnit assert if inTestingContext, otherwise checks if b is true and - * throws an exception if not. - * - * @param b The boolean checked - */ - private static void checkTrue(boolean b) throws Exception { - if (inTestingContext) { - assertTrue(b); - } else if (!b) { - throw new Exception("Failed assertion"); - } - } - - /** - * Checks if l1 and l2 are equal. Uses JUnit assert if inTestingContext, otherwise checks if l1 - * and l2 are equal and throws an exception if not. - * - * @param l1 One of the two longs to be compared - * @param l2 One of the two longs to be compared - */ - private static void checkEquals(long l1, long l2) throws Exception { - if (inTestingContext) { - assertEquals(l1, l2); - } else if (l1 != l2) { - throw new Exception("Failed assertion"); - } - } - /** * Tracks what data has been written and deleted in an Accumulo table. */ @@ -412,7 +385,7 @@ private static ScanStats scan(Stream> scanner, Set exp } }); - checkTrue(expected.isEmpty()); + assertTrue(expected.isEmpty()); return stats; } From 14c2b4af9312f20893911a3a7a80dea48f0132c2 Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Tue, 12 Dec 2023 09:51:39 -0500 Subject: [PATCH 06/11] scan() no longer throws Exception --- .../main/java/org/apache/accumulo/test/ScanConsistencyIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index 0d125bc7ba0..9194e7d80a4 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -372,8 +372,7 @@ public void add(ScanStats stats) { } } - private static ScanStats scan(Stream> scanner, Set expected) - throws Exception { + private static ScanStats scan(Stream> scanner, Set expected) { ScanStats stats = new ScanStats(); scanner.forEach(entry -> { From fe2f2170221800f4ae299d6501e54ddf7796ea4a Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Tue, 12 Dec 2023 10:30:00 -0500 Subject: [PATCH 07/11] Moved SuppressFBWarnings to correct method --- .../main/java/org/apache/accumulo/test/ScanConsistencyIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index 9194e7d80a4..3f04f151f65 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -117,8 +117,6 @@ public static void main(String[] args) { } } - @SuppressFBWarnings(value = {"PREDICTABLE_RANDOM", "DMI_RANDOM_USED_ONLY_ONCE"}, - justification = "predictable random is ok for testing") @Test public void testConcurrentScanConsistency() throws Exception { try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { @@ -130,6 +128,8 @@ public void testConcurrentScanConsistency() throws Exception { } } + @SuppressFBWarnings(value = {"PREDICTABLE_RANDOM", "DMI_RANDOM_USED_ONLY_ONCE"}, + justification = "predictable random is ok for testing") private static void runTest(AccumuloClient client, FileSystem fileSystem, String tmpDir, String table, long sleepTime) throws Exception { From b398bee2b82b54943e8cc0c05d434eac6f2e4af7 Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Tue, 12 Dec 2023 10:42:27 -0500 Subject: [PATCH 08/11] Formatting fix --- .../main/java/org/apache/accumulo/test/ScanConsistencyIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index 3f04f151f65..f701ca5a040 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -129,7 +129,7 @@ public void testConcurrentScanConsistency() throws Exception { } @SuppressFBWarnings(value = {"PREDICTABLE_RANDOM", "DMI_RANDOM_USED_ONLY_ONCE"}, - justification = "predictable random is ok for testing") + justification = "predictable random is ok for testing") private static void runTest(AccumuloClient client, FileSystem fileSystem, String tmpDir, String table, long sleepTime) throws Exception { From 4fe6fda0fcca1b45ec8bdf9833b3db7fe2ccc796 Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Tue, 19 Dec 2023 10:41:21 -0500 Subject: [PATCH 09/11] Updated javadoc for main, main now throws exception --- .../accumulo/test/ScanConsistencyIT.java | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index f701ca5a040..837314c085b 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -80,30 +80,34 @@ public class ScanConsistencyIT extends AccumuloClusterHarness { private static final Logger log = LoggerFactory.getLogger(ScanConsistencyIT.class); - public static void main(String[] args) { - /** - * @formatter:off - * Note: In order to run main, - * 1) Build the project - * 2) Copy the accumulo test jar (in /test/target/) into your accumulo installation's - * lib directory* - * 3) Copy the JUnit dependencies into your accumulo installation's lib directory: - * "mvn dependency:copy-dependencies -DincludeGroupIds="org.junit.jupiter" - * cp test/target/dependency/junit-jupiter-* $ACCUMULO_HOME/lib/" - * - * *Ensure the test jar is in lib before the tablet servers start. Restart tablet - * servers if necessary. - * - * Now, this can be run with - * "accumulo org.apache.accumulo.test.ScanConsistencyIT
" - * : An accumulo client properties file - * : tmpDir field for the TestContext object - *
: The name of the table to be created - * : The time to sleep (ms) after submitting the various concurrent tasks - * @formatter:on - */ + /** + * Note: In order to run main,
+ * 1) Build the project
+ * 2) Copy the accumulo test jar (in /test/target/) into your accumulo installation's lib + * directory*
+ * 3) Copy the JUnit dependencies into your accumulo installation's lib directory:
+ * $ mvn dependency:copy-dependencies -DincludeGroupIds="org.junit.jupiter"
+ * $ cp test/target/dependency/junit-jupiter-* $ACCUMULO_HOME/lib/
+ *
+ * *Ensure the test jar is in lib before the tablet servers start. Restart tablet servers if + * necessary.
+ *
+ * Now, this can be run with
+ * $ accumulo org.apache.accumulo.test.ScanConsistencyIT [props-file] [tmp-dir] [table] + * [sleep-time]
+ *
+ * + * [props-file]: An accumulo client properties file
+ * [tmp-dir]: tmpDir field for the TestContext object
+ * [table]: The name of the table to be created
+ * [sleep-time]: The time to sleep (ms) after submitting the various concurrent tasks
+ *
+ * + * @param args The props file, temp directory, table, and sleep time + */ + public static void main(String[] args) throws Exception { Preconditions.checkArgument(args.length == 4, "Invalid arguments. Use: " - + "accumulo org.apache.accumulo.test.ScanConsistencyIT
"); + + "accumulo org.apache.accumulo.test.ScanConsistencyIT [props-file] [tmp-dir] [table] [sleep-time]"); final String propsFile = args[0]; final String tmpDir = args[1]; final String table = args[2]; @@ -112,8 +116,6 @@ public static void main(String[] args) { try (AccumuloClient client = Accumulo.newClient().from(propsFile).build()) { FileSystem fileSystem = FileSystem.get(new Configuration()); runTest(client, fileSystem, tmpDir, table, sleepTime); - } catch (Exception e) { - log.error(e.toString()); } } From 6e04d162f3bbe32c73b61ddb8d3353236fdffddf Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Tue, 19 Dec 2023 11:41:04 -0500 Subject: [PATCH 10/11] Added code tags to javadocs --- .../java/org/apache/accumulo/test/ScanConsistencyIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index 837314c085b..0a88caae286 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -86,16 +86,18 @@ public class ScanConsistencyIT extends AccumuloClusterHarness { * 2) Copy the accumulo test jar (in /test/target/) into your accumulo installation's lib * directory*
* 3) Copy the JUnit dependencies into your accumulo installation's lib directory:
+ * * $ mvn dependency:copy-dependencies -DincludeGroupIds="org.junit.jupiter"
* $ cp test/target/dependency/junit-jupiter-* $ACCUMULO_HOME/lib/
- *
+ *
* *Ensure the test jar is in lib before the tablet servers start. Restart tablet servers if * necessary.
*
* Now, this can be run with
+ * * $ accumulo org.apache.accumulo.test.ScanConsistencyIT [props-file] [tmp-dir] [table] * [sleep-time]
- *
+ *
* * [props-file]: An accumulo client properties file
* [tmp-dir]: tmpDir field for the TestContext object
From 3ea4e835468e787105890ec5cbdc34288f949580 Mon Sep 17 00:00:00 2001 From: Kevin Rathbun Date: Wed, 20 Dec 2023 10:21:42 -0500 Subject: [PATCH 11/11] Reformatted javadoc for main --- .../accumulo/test/ScanConsistencyIT.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java index 0a88caae286..cbdece46feb 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java @@ -81,23 +81,19 @@ public class ScanConsistencyIT extends AccumuloClusterHarness { private static final Logger log = LoggerFactory.getLogger(ScanConsistencyIT.class); /** - * Note: In order to run main,
- * 1) Build the project
- * 2) Copy the accumulo test jar (in /test/target/) into your accumulo installation's lib - * directory*
- * 3) Copy the JUnit dependencies into your accumulo installation's lib directory:
- * - * $ mvn dependency:copy-dependencies -DincludeGroupIds="org.junit.jupiter"
- * $ cp test/target/dependency/junit-jupiter-* $ACCUMULO_HOME/lib/
- *
- * *Ensure the test jar is in lib before the tablet servers start. Restart tablet servers if - * necessary.
- *
- * Now, this can be run with
- * - * $ accumulo org.apache.accumulo.test.ScanConsistencyIT [props-file] [tmp-dir] [table] - * [sleep-time]
- *
+ * Note: In order to run main, + *
    + *
  1. Build the project
    2. + *
  2. Copy the accumulo test jar (in /test/target/) into your accumulo installation's lib + * directory
    3. + *
  3. Copy the JUnit dependencies into your accumulo installation's lib directory: mvn + * dependency:copy-dependencies -DincludeGroupIds="org.junit.jupiter" and cp + * test/target/dependency/junit-jupiter-* $ACCUMULO_HOME/lib/
    4. + *
  4. Ensure the test jar is in lib before the tablet servers start. Restart tablet servers if + * necessary.
    5. + *
  5. Run with: accumulo org.apache.accumulo.test.ScanConsistencyIT [props-file] [tmp-dir] + * [table] [sleep-time]
    6. + *
* * [props-file]: An accumulo client properties file
* [tmp-dir]: tmpDir field for the TestContext object