diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java index b2e1aaf539ab8..2c6bd93e0f5fc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java @@ -63,7 +63,7 @@ public void testBasicAlterPipe() throws Exception { // The database & table name will be converted to lower case final String sql = String.format( - "create pipe a2b with source ('source'='iotdb-source', 'database-name'='Test', 'table-name'='Test1', 'mode.streaming'='true') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", + "create pipe a2b with source ('source'='iotdb-source', 'database-name'='Test', 'table-name'='Test1', 'source.realtime.mode'='batch') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { @@ -88,7 +88,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -146,7 +146,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test1")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test1")); - Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -185,7 +185,8 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("database-name=test")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("table-name=test")); - Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("mode.streaming=true")); + Assert.assertFalse( + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -443,7 +444,7 @@ public void testAlterPipeFailure() { // Create pipe sql = String.format( - "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test1', 'mode.streaming'='true') with sink ('node-urls'='%s', 'batch.enable'='false')", + "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test1', 'source.realtime.mode'='batch') with sink ('node-urls'='%s', 'batch.enable'='false')", receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { @@ -468,7 +469,7 @@ public void testAlterPipeSourceAndSink() { // Create pipe final String sql = String.format( - "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test', 'mode.streaming'='true') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s', 'batch.enable'='false')", + "create pipe a2b with source ('source'='iotdb-source', 'database-name'='test', 'table-name'='test', 'source.realtime.mode'='batch') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s', 'batch.enable'='false')", receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { @@ -488,7 +489,7 @@ public void testAlterPipeSourceAndSink() { try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "alter pipe a2b modify source('source' = 'iotdb-source','database-name'='test1', 'table-name'='test1', 'mode.streaming'='true', 'source.inclusion'='data.insert') modify sink ('batch.enable'='true')"); + "alter pipe a2b modify source('source' = 'iotdb-source','database-name'='test1', 'table-name'='test1', 'source.realtime.mode'='batch', 'source.inclusion'='data.insert') modify sink ('batch.enable'='true')"); } catch (final SQLException e) { fail(e.getMessage()); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java index 2a5c9352e2542..f2b5824b29528 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java @@ -77,6 +77,7 @@ public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TestUtils.executeNonQueries( senderEnv, @@ -87,7 +88,7 @@ public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("source.realtime.mode", "log"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("capture.table", "true"); sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); @@ -112,6 +113,7 @@ public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); TableModelUtils.insertData("test", "test", 50, 100, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TestUtils.executeNonQueries( senderEnv, @@ -144,11 +146,6 @@ public void testTsFileFormatAndAsyncLoad() throws Exception { testSinkFormat("tsfile", true); } - @Test - public void testSinkHybridFormat() throws Exception { - testSinkFormat("hybrid", false); - } - private void testSinkFormat(final String format, final boolean isAsyncLoad) throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -165,6 +162,7 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TestUtils.executeNonQueries( senderEnv, @@ -202,6 +200,7 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); TableModelUtils.insertData("test", "test", 50, 150, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TestUtils.executeNonQueries( senderEnv, @@ -244,6 +243,7 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro null); TableModelUtils.insertData("test", "test", 150, 200, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TableModelUtils.insertTablet("test", "test", 200, 250, senderEnv, true); TableModelUtils.insertTablet("test", "test", 250, 300, senderEnv, true); TableModelUtils.insertTablet("test", "test", 300, 350, senderEnv, true); @@ -272,8 +272,8 @@ public void testWriteBackSink() throws Exception { sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); sourceAttributes.put("mode.double-living", "true"); - sourceAttributes.put("forwarding-pipe-requests", "false"); - sourceAttributes.put("source.database-name", "test.*"); + sourceAttributes.put("source.realtime.mode", "batch"); + sourceAttributes.put("source.database-name", "test"); sourceAttributes.put("source.table-name", "test.*"); sourceAttributes.put("user", "root"); @@ -299,6 +299,7 @@ public void testWriteBackSink() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertDataNotThrowError("test", "test", 0, 20, senderEnv); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TableModelUtils.insertTablet("test", "test", 20, 200, senderEnv, true); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java index afa6b38775e62..9921700527c14 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java @@ -204,7 +204,7 @@ public void testLifeCycleLogMode() throws Exception { sourceAttributes.put("capture.table", "true"); sourceAttributes.put("__system.sql-dialect", "table"); - sourceAttributes.put("source.mode", "forced-log"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -261,7 +261,7 @@ public void testLifeCycleFileMode() throws Exception { sourceAttributes.put("capture.table", "true"); sourceAttributes.put("__system.sql-dialect", "table"); - sourceAttributes.put("mode.streaming", "false"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -292,59 +292,6 @@ public void testLifeCycleFileMode() throws Exception { } } - @Test - public void testLifeCycleHybridMode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - final Consumer handleFailure = - o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); - }; - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 100, senderEnv); - - final Map sourceAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map sinkAttributes = new HashMap<>(); - - sourceAttributes.put("capture.table", "true"); - sourceAttributes.put("__system.sql-dialect", "table"); - sourceAttributes.put("source.mode", "hybrid"); - sourceAttributes.put("user", "root"); - - sinkAttributes.put("sink", "iotdb-thrift-sink"); - sinkAttributes.put("sink.batch.enable", "false"); - sinkAttributes.put("sink.ip", receiverIp); - sinkAttributes.put("sink.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", sinkAttributes) - .setExtractorAttributes(sourceAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TableModelUtils.insertData("test", "test", 100, 200, senderEnv); - - TableModelUtils.assertCountData("test", "test", 200, receiverEnv, handleFailure); - - TableModelUtils.insertData("test", "test", 200, 300, senderEnv); - - TableModelUtils.assertCountData("test", "test", 300, receiverEnv, handleFailure); - } - } - @Test public void testLifeCycleWithClusterRestart() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -544,6 +491,7 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { } @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLiving() throws Exception { // Double living is two clusters with pipes connecting each other. diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java index 9a6fe6ddf41d1..cf2ac218c1bd1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java @@ -219,8 +219,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("source.realtime.mode", "batch"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("user", "root"); @@ -263,8 +262,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { extractorAttributes.put("database-name", "test.*"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("source.realtime.mode", "batch"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("user", "root"); @@ -313,8 +311,7 @@ private void doTest() throws Exception { extractorAttributes.put("database-name", "test.*"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("source.realtime.mode", "batch"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("user", "root"); @@ -358,11 +355,6 @@ public void testAsyncConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); } - @Test - public void testAirGapConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); - } - private void doTestUseNodeUrls(String connectorName) throws Exception { senderEnv .getConfig() @@ -407,16 +399,7 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { }; for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { - if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { - // Use default port for convenience - nodeUrlsBuilder - .append(wrapper.getIp()) - .append(":") - .append(wrapper.getPipeAirGapReceiverPort()) - .append(","); - } else { - nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); - } + nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); } try (final SyncConfigNodeIServiceClient client = @@ -440,12 +423,11 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { extractorAttributes.put("database-name", "test.*"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.snapshot", "false"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("user", "root"); - // Test forced-log mode, in open releases this might be "file" - extractorAttributes.put("realtime.mode", "forced-log"); + // Test batch mode + extractorAttributes.put("realtime.mode", "batch"); TSStatus status = client.createPipe( @@ -465,7 +447,7 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { TableModelUtils.assertData("test", "test", 0, 300, receiverEnv, handleFailure); - extractorAttributes.replace("realtime.mode", "file"); + extractorAttributes.replace("realtime.mode", "batch"); status = client.createPipe( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java index a759a714a7faf..48ffd86658e7e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java @@ -71,8 +71,7 @@ public void testPipeSwitchStatus() throws Exception { extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("source.realtime.mode", "batch"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("start-time", "1"); extractorAttributes.put("end-time", "2"); @@ -170,8 +169,7 @@ public void testPipeIllegallySwitchStatus() throws Exception { extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("source.realtime.mode", "batch"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("start-time", "1"); extractorAttributes.put("end-time", "2"); @@ -257,8 +255,7 @@ public void testDropPipeAndCreateAgain() throws Exception { extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("source.realtime.mode", "batch"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("start-time", "0"); extractorAttributes.put("end-time", "200"); @@ -325,8 +322,7 @@ public void testWrongPipeName() throws Exception { extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); - extractorAttributes.put("mode.streaming", "true"); - extractorAttributes.put("mode.snapshot", "false"); + extractorAttributes.put("source.realtime.mode", "batch"); extractorAttributes.put("mode.strict", "true"); extractorAttributes.put("start-time", "0"); extractorAttributes.put("end-time", "200"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java index ec1afaa43eea6..a3c35dc978cfe 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java @@ -82,7 +82,7 @@ public void testValidPipeName() throws Exception { + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," - + "'mode.streaming'='true'," + + "'source.realtime.mode'='batch'," + "'mode.strict'='true'," + "'mode.snapshot'='false'," + "'start-time'='1'," @@ -145,7 +145,7 @@ public void testRevertParameterOrder() { + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," - + "'mode.streaming'='true'," + + "'source.realtime.mode'='batch'," + "'mode.strict'='true'," + "'mode.snapshot'='false'," + "'start-time'='1'," @@ -185,7 +185,7 @@ public void testRevertStageOrder() throws Exception { + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," - + "'mode.streaming'='true'," + + "'source.realtime.mode'='batch'," + "'mode.strict'='true'," + "'mode.snapshot'='false'," + "'start-time'='1'," @@ -402,7 +402,7 @@ public void testBrackets() throws Exception { + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," - + "'mode.streaming'='true'," + + "'source.realtime.mode'='batch'," + "'mode.strict'='true'," + "'mode.snapshot'='false'," + "'start-time'='1'," @@ -429,7 +429,7 @@ public void testBrackets() throws Exception { + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," - + "'mode.streaming'='true'," + + "'source.realtime.mode'='batch'," + "'mode.strict'='true'," + "'mode.snapshot'='false'," + "'start-time'='1'," @@ -454,7 +454,7 @@ public void testBrackets() throws Exception { + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," - + "'mode.streaming'='true'," + + "'source.realtime.mode'='batch'," + "'mode.strict'='true'," + "'mode.snapshot'='false'," + "'start-time'='1'," @@ -579,7 +579,7 @@ public void testShowPipeWithWrongPipeName() throws Exception { extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.mode.streaming", "true"); + extractorAttributes.put("extractor.realtime.mode", "batch"); extractorAttributes.put("extractor.mode.strict", "true"); extractorAttributes.put("extractor.start-time", "1"); extractorAttributes.put("extractor.end-time", "2"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java index d635a9a41805d..1c927c5d4968c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java @@ -155,7 +155,7 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("extractor.realtime.mode", "batch"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector.batch.enable", "false"); @@ -240,7 +240,7 @@ public void testReceiverAutoExtendIdColumn() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("extractor.realtime.mode", "batch"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector.batch.enable", "false"); @@ -322,7 +322,7 @@ public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("extractor.realtime.mode", "batch"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector.batch.enable", "false"); @@ -396,7 +396,7 @@ public void testLoadAutoCreateWithTableDeletion() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("extractor.realtime.mode", "batch"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector.batch.enable", "false"); @@ -446,7 +446,7 @@ public void testLoadAutoCreateWithoutInsertPermission() throws Exception { extractorAttributes.put("capture.table", "true"); extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("extractor.realtime.mode", "file"); + extractorAttributes.put("extractor.realtime.mode", "batch"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector.batch.enable", "false"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java index b86646de0d938..b553cfc842d73 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java @@ -35,6 +35,7 @@ import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -83,6 +84,7 @@ public void setUp() { } @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLivingAutoConflict() throws Exception { // Double living is two clusters each with a pipe connecting to the other. final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); @@ -184,6 +186,7 @@ public void testDoubleLivingAutoConflict() throws Exception { } @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLivingAutoConflictTemplate() throws Exception { final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java index c99eb5d941a16..097a1fc5c4038 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java @@ -184,12 +184,12 @@ private void testMachineDowntime(String sink) { @Test public void testWithAllParametersInStreamingMode() throws Exception { - testWithAllParameters("true"); + testWithAllParameters("batch"); } @Test public void testWithAllParametersInNotStreamingMode() throws Exception { - testWithAllParameters("false"); + testWithAllParameters("batch"); } private void testWithAllParameters(final String realtimeMode) throws Exception { @@ -220,7 +220,7 @@ private void testWithAllParameters(final String realtimeMode) throws Exception { sourceAttributes.put("table-name", "test"); sourceAttributes.put("start-time", "0"); sourceAttributes.put("end-time", "199"); - sourceAttributes.put("mode.streaming", realtimeMode); + sourceAttributes.put("source.realtime.mode", realtimeMode); sourceAttributes.put("user", "root"); processorAttributes.put("processor", "do-nothing-processor"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java index 26f7dd9db9ca5..d9ebd6f49b0ee 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java @@ -36,6 +36,7 @@ import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -120,6 +121,7 @@ public void testDoubleLivingInvalidForwardingParameter() throws Exception { // org.apache.iotdb.pipe.it.tablemodel.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving and // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testBasicDoubleLiving() { final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java index e56e0e9e47b39..4f506df55403c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java @@ -229,7 +229,6 @@ private void testTableConfigIdempotent(final List beforeSqlList, final S sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("source.capture.table", "true"); sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("source.capture.tree", "false"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java index 5470b61c368cc..b87d92219d3cd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java @@ -81,6 +81,7 @@ private void testInsertNullValueTemplate( TableModelUtils.insertTablet("test", "test", 0, 200, senderEnv, true); } else if (insertType == InsertType.SQL_INSERT) { TableModelUtils.insertData("test", "test", 0, 200, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); } connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -110,6 +111,7 @@ private void testInsertNullValueTemplate( TableModelUtils.insertTablet("test", "test", 200, 400, senderEnv, true); } else if (insertType == InsertType.SQL_INSERT) { TableModelUtils.insertData("test", "test", 200, 400, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); } if (withParsing) { @@ -124,32 +126,32 @@ private void testInsertNullValueTemplate( // ---------------------- // @Test public void testSQLInsertWithParsingForcedLog() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "forced-log"); + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "batch"); } @Test public void testSQLInsertWithoutParsingForcedLog() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "forced-log"); + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "batch"); } @Test public void testSQLInsertWithParsingFile() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "file"); + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "batch"); } @Test public void testSQLInsertWithoutParsingFile() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "file"); + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "batch"); } @Test public void testSQLInsertWithParsingStream() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "stream"); + testInsertNullValueTemplate(InsertType.SQL_INSERT, true, "batch"); } @Test public void testSQLInsertWithoutParsingStream() throws Exception { - testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "stream"); + testInsertNullValueTemplate(InsertType.SQL_INSERT, false, "batch"); } // --------------------------------- // @@ -157,31 +159,31 @@ public void testSQLInsertWithoutParsingStream() throws Exception { // --------------------------------- // @Test public void testSessionInsertTabletWithParsingForcedLog() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "forced-log"); + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "batch"); } @Test public void testSessionInsertTabletWithoutParsingForcedLog() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "forced-log"); + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "batch"); } @Test public void testSessionInsertTabletWithParsingFile() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "file"); + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "batch"); } @Test public void testSessionInsertTabletWithoutParsingFile() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "file"); + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "batch"); } @Test public void testSessionInsertTabletWithParsingStream() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "stream"); + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, true, "batch"); } @Test public void testSessionInsertTabletWithoutParsingStream() throws Exception { - testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "stream"); + testInsertNullValueTemplate(InsertType.SESSION_INSERT_TABLET, false, "batch"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java index fe2b854863da7..2edcca121e801 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java @@ -62,7 +62,6 @@ public class IoTDBPipeSinkCompressionIT extends AbstractPipeTableModelDualManual @Override @Before public void setUp() { - // Override to enable air-gap MultiEnvFactory.createEnv(2); senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); @@ -98,7 +97,7 @@ public void setUp() { @Test public void testCompression1() throws Exception { - doTest("iotdb-thrift-connector", "stream", true, "snappy"); + doTest("iotdb-thrift-connector", "batch", true, "snappy"); } @Test @@ -108,7 +107,7 @@ public void testCompression2() throws Exception { @Test public void testCompression3() throws Exception { - doTest("iotdb-thrift-sync-connector", "stream", false, "snappy, snappy"); + doTest("iotdb-thrift-sync-connector", "batch", false, "snappy, snappy"); } @Test @@ -116,26 +115,13 @@ public void testCompression4() throws Exception { doTest("iotdb-thrift-sync-connector", "batch", true, "gzip, zstd"); } - @Test - public void testCompression5() throws Exception { - doTest("iotdb-air-gap-connector", "stream", false, "lzma2, lz4"); - } - - @Test - public void testCompression6() throws Exception { - doTest("iotdb-air-gap-connector", "batch", true, "lzma2"); - } - private void doTest( String connectorType, String realtimeMode, boolean useBatchMode, String compressionTypes) throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = - connectorType.contains("air-gap") - ? receiverDataNode.getPipeAirGapReceiverPort() - : receiverDataNode.getPort(); + final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { @@ -145,6 +131,7 @@ private void doTest( TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); TableModelUtils.insertData("test", "test", 0, 50, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -207,6 +194,7 @@ private void doTest( null); TableModelUtils.insertData("test", "test", 50, 100, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -348,12 +336,15 @@ public void testZstdCompressorLevel() throws Exception { TableModelUtils.createDataBaseAndTable(senderEnv, "test1", "test"); TableModelUtils.insertData("test", "test1", 0, 50, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TableModelUtils.createDataBaseAndTable(senderEnv, "test2", "test"); TableModelUtils.insertData("test", "test2", 0, 50, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TableModelUtils.createDataBaseAndTable(senderEnv, "test3", "test"); TableModelUtils.insertData("test", "test3", 0, 50, senderEnv, true); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); TableModelUtils.assertCountData("test", "test1", 50, receiverEnv, handleFailure); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java index 2da24a820e017..7c8bd81deb142 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java @@ -200,7 +200,7 @@ private void createDataPipe(boolean isTSFile) { + " with source ('source'='iotdb-source','realtime.mode'='%s','history.enable'='%s')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", - isTSFile ? "file" : "forced-log", + "batch", isTSFile, receiverEnv.getIP(), receiverEnv.getPort(), diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java index 68adc1226137e..cc7f8c0a4d42d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java @@ -259,7 +259,7 @@ private void createDataPipe() { String sql = String.format( "create pipe test" - + " with source ('source'='iotdb-source','realtime.mode'='forced-log','realtime.enable'='true','history.enable'='false')" + + " with source ('source'='iotdb-source','realtime.mode'='batch','realtime.enable'='true','history.enable'='false')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='tablet')", receiverEnv.getIP(), receiverEnv.getPort()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java index 6704d37b87e2e..d81acd751cbc3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionSemanticIT.java @@ -156,7 +156,7 @@ private void createPipe() { senderEnv, String.format( "create pipe type_conversion_semantic" - + " with source ('source'='iotdb-source','history.enable'='false','realtime.enable'='true','realtime.mode'='forced-log')" + + " with source ('source'='iotdb-source','history.enable'='false','realtime.enable'='true','realtime.mode'='batch')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s','batch.enable'='false','sink.format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()), @@ -170,7 +170,7 @@ private void createStreamPipe() { senderEnv, String.format( "create pipe stream_type_conversion_semantic" - + " with source ('source'='iotdb-source','history.enable'='false','realtime.enable'='true','realtime.mode'='stream')" + + " with source ('source'='iotdb-source','history.enable'='false','realtime.enable'='true','realtime.mode'='batch')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('sink'='iotdb-thrift-sink','sink.node-urls'='%s')", receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()), diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java index d32bc33c956e3..fd186514beacb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java @@ -40,9 +40,7 @@ import java.sql.Statement; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static org.junit.Assert.fail; @@ -63,7 +61,7 @@ public void testBasicAlterPipe() throws Exception { // Create pipe final String sql = String.format( - "create pipe a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='stream') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", + "create pipe a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='batch') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -85,8 +83,7 @@ public void testBasicAlterPipe() throws Exception { // Check configurations Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -139,8 +136,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -178,7 +174,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test2")); Assert.assertFalse( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -240,7 +236,7 @@ public void testBasicAlterPipe() throws Exception { try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { statement.execute( - "alter pipe a2b replace processor ('processor'='tumbling-time-sampling-processor')"); + "alter pipe a2b replace processor ('processor'='iot-consensus-v2-processor')"); } catch (SQLException e) { fail(e.getMessage()); } @@ -258,10 +254,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); + showPipeResult.get(0).pipeProcessor.contains("processor=iot-consensus-v2-processor")); Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=false")); Assert.assertTrue( showPipeResult @@ -297,10 +290,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); + showPipeResult.get(0).pipeProcessor.contains("processor=iot-consensus-v2-processor")); Assert.assertTrue( showPipeResult .get(0) @@ -335,10 +325,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); + showPipeResult.get(0).pipeProcessor.contains("processor=iot-consensus-v2-processor")); Assert.assertTrue( showPipeResult .get(0) @@ -373,10 +360,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); Assert.assertTrue( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); + showPipeResult.get(0).pipeProcessor.contains("processor=iot-consensus-v2-processor")); Assert.assertTrue( showPipeResult .get(0) @@ -411,10 +395,7 @@ public void testBasicAlterPipe() throws Exception { Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.path=root.test1.**")); Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); Assert.assertFalse( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); + showPipeResult.get(0).pipeProcessor.contains("processor=iot-consensus-v2-processor")); Assert.assertTrue( showPipeResult .get(0) @@ -447,10 +428,7 @@ public void testBasicAlterPipe() throws Exception { // Check configurations Assert.assertTrue(showPipeResult.get(0).pipeConnector.contains("batch.enable=true")); Assert.assertFalse( - showPipeResult - .get(0) - .pipeProcessor - .contains("processor=tumbling-time-sampling-processor")); + showPipeResult.get(0).pipeProcessor.contains("processor=iot-consensus-v2-processor")); Assert.assertTrue( showPipeResult .get(0) @@ -492,85 +470,6 @@ public void testAlterPipeFailure() { } } - @Test - public void testAlterPipeSourceAndProcessor() { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - // Create pipe - final String sql = - String.format( - "create pipe a2b with source ('source' = 'iotdb-source','source.path' = 'root.db.d1.**') with processor ('processor'='tumbling-time-sampling-processor', 'processor.tumbling-time.interval-seconds'='1', 'processor.down-sampling.split-file'='true') with sink ('node-urls'='%s', 'batch.enable'='false')", - receiverDataNode.getIpAndPortString()); - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute(sql); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Insert data on sender - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (1000, 1), (1500, 2), (2000, 3), (2500, 4), (3000, 5)", - "flush"), - null); - - // Check data on receiver - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1000,1.0,"); - expectedResSet.add("2000,3.0,"); - expectedResSet.add("3000,5.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.at1,", expectedResSet); - - // Alter pipe (modify 'source.path', 'source.inclusion' and - // 'processor.tumbling-time.interval-seconds') - try (final Connection connection = senderEnv.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute( - "alter pipe a2b modify source('source' = 'iotdb-source','source.path'='root.db.d2.**', 'source.inclusion'='all') modify processor ('processor.tumbling-time.interval-seconds'='2')"); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Insert data on sender - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - "insert into root.db.d2 (time, at1) values (11000, 1), (11500, 2), (12000, 3), (12500, 4), (13000, 5)", - "flush"), - null); - - // Insert data on sender - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - "insert into root.db.d1 (time, at1) values (11000, 1), (11500, 2), (12000, 3), (12500, 4), (13000, 5)", - "flush"), - null); - - // Check data on receiver - expectedResSet.clear(); - expectedResSet.add("11000,null,1.0,"); - expectedResSet.add("13000,null,5.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select * from root.db.** where time > 10000", - "Time,root.db.d1.at1,root.db.d2.at1,", - expectedResSet); - - // Create database on sender - TestUtils.executeNonQuery(senderEnv, "create timeSeries root.db.d2.at2 int32", null); - - // Check database on receiver - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "count timeSeries root.db.**", - "count(timeseries),", - Collections.singleton("3,")); - } - @Test public void testAlterPipeRealtime() { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java index 03835e7743bf3..95c7df55e82dc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java @@ -89,7 +89,7 @@ public void testThriftSinkWithRealtimeFirstDisabled() throws Exception { final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("source.realtime.mode", "log"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -151,7 +151,7 @@ private void testSinkFormat(final String format) throws Exception { final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("source.realtime.mode", "forced-log"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -251,7 +251,7 @@ public void testReceiverAutoCreateByLog() throws Exception { testReceiverAutoCreate( new HashMap() { { - put("source.realtime.mode", "forced-log"); + put("source.realtime.mode", "batch"); put("user", "root"); } }); @@ -390,7 +390,7 @@ private void testReceiverLoadTsFile(final String loadTsFileStrategy) throws Exce final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("source.realtime.mode", "forced-log"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -579,7 +579,7 @@ public void testTransferMods() { "flush", "delete timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.s0", String.format( - "create pipe test with source ('source.realtime.mode'='stream','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", + "create pipe test with source ('source.realtime.mode'='batch','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()))); TestUtils.assertDataEventuallyOnEnv( @@ -599,7 +599,7 @@ public void testTransferMods() { Arrays.asList( "delete timeseries root.sg_nonAligned.非对齐序列带有encoding和压缩方式.s1", String.format( - "create pipe test with source ('source.realtime.mode'='stream','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", + "create pipe test with source ('source.realtime.mode'='batch','inclusion'='data','path'='root.sg_nonAligned.非对齐序列带有encoding和压缩方式.**','source.realtime.enable'='true','mods.enable'='true') with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')", receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()))); TestUtils.assertDataEventuallyOnEnv( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java index 188f2813115c0..e6df830232672 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java @@ -32,6 +32,7 @@ import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -281,7 +282,7 @@ public void testLifeCycleLogMode() throws Exception { final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("source.mode", "log"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -346,72 +347,7 @@ public void testLifeCycleFileMode() throws Exception { final Map processorAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("source.mode", "file"); - sourceAttributes.put("user", "root"); - - sinkAttributes.put("sink", "iotdb-thrift-sink"); - sinkAttributes.put("sink.batch.enable", "false"); - sinkAttributes.put("sink.ip", receiverIp); - sinkAttributes.put("sink.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", sinkAttributes) - .setExtractorAttributes(sourceAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - final Set expectedResSet = new HashSet<>(); - expectedResSet.add("1,1.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList("insert into root.db.d1(time, s1) values (2, 2)", "flush"), - null); - - expectedResSet.add("2,2.0,"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipe("p1").getCode()); - - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList("insert into root.db.d1(time, s1) values (3, 3)", "flush"), - null); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.db.**", "Time,root.db.d1.s1,", expectedResSet); - } - } - - @Test - public void testLifeCycleHybridMode() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList("insert into root.db.d1(time, s1) values (1, 1)", "flush"), - null); - - final Map sourceAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map sinkAttributes = new HashMap<>(); - - sourceAttributes.put("source.mode", "hybrid"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -669,6 +605,7 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { } @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLiving() throws Exception { // Double living is two clusters with pipes connecting each other. final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java deleted file mode 100644 index d17d73516fa74..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.consensus.ConsensusFactory; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeProcessorIT extends AbstractPipeDualTreeModelAutoIT { - - @Before - public void setUp() { - MultiEnvFactory.createEnv(2); - senderEnv = MultiEnvFactory.getEnv(0); - receiverEnv = MultiEnvFactory.getEnv(1); - - senderEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setDatanodeMemoryProportion("3:3:1:1:1:0") - .setTimestampPrecision("ms") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setDatanodeMemoryProportion("3:3:1:1:1:0") - .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) - .setPipeMemoryManagementEnabled(false) - .setIsPipeEnableMemoryCheck(false); - - // 10 min, assert that the operations will not time out - senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); - - senderEnv.initClusterEnvironment(); - receiverEnv.initClusterEnvironment(); - } - - @Ignore - @Test - public void testTumblingTimeSamplingProcessor() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // Test empty tsFile parsing - // Assert that an empty tsFile will not be parsed by the processor then block - // the subsequent data processing - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - "insert into root.vehicle.d0(time, s1) values (0, 1)", "delete from root.**"), - null); - - final Map sourceAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map sinkAttributes = new HashMap<>(); - - sourceAttributes.put("source.realtime.mode", "log"); - sourceAttributes.put("user", "root"); - - processorAttributes.put("processor", "tumbling-time-sampling-processor"); - processorAttributes.put("processor.tumbling-time.interval-seconds", "20"); - processorAttributes.put("processor.down-sampling.split-file", "true"); - - sinkAttributes.put("sink", "iotdb-thrift-sink"); - sinkAttributes.put("sink.batch.enable", "false"); - sinkAttributes.put("sink.ip", receiverIp); - sinkAttributes.put("sink.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(sourceAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - TestUtils.executeNonQueries( - senderEnv, - Arrays.asList( - "insert into root.vehicle.d0(time, s1) values (0, 1)", - "insert into root.vehicle.d0(time, s1) values (10000, 2)", - "insert into root.vehicle.d0(time, s1) values (19999, 3)", - "insert into root.vehicle.d0(time, s1) values (20000, 4)", - "insert into root.vehicle.d0(time, s1) values (20001, 5)", - "insert into root.vehicle.d0(time, s1) values (45000, 6)", - "flush"), - null); - - final Set expectedResSet = new HashSet<>(); - - expectedResSet.add("0,1.0,"); - expectedResSet.add("20000,4.0,"); - expectedResSet.add("45000,6.0,"); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "select * from root.vehicle.**", "Time,root.vehicle.d0.s1,", expectedResSet); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java index 92874b6d5397b..f3b7612c5da59 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java @@ -349,11 +349,6 @@ public void testAsyncConnectorUseNodeUrls() throws Exception { doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); } - @Test - public void testAirGapConnectorUseNodeUrls() throws Exception { - doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); - } - private void doTestUseNodeUrls(String sinkName) throws Exception { senderEnv .getConfig() @@ -390,16 +385,7 @@ private void doTestUseNodeUrls(String sinkName) throws Exception { final StringBuilder nodeUrlsBuilder = new StringBuilder(); for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) { - if (sinkName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) { - // Use default port for convenience - nodeUrlsBuilder - .append(wrapper.getIp()) - .append(":") - .append(wrapper.getPipeAirGapReceiverPort()) - .append(","); - } else { - nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); - } + nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(","); } try (final SyncConfigNodeIServiceClient client = @@ -427,8 +413,8 @@ private void doTestUseNodeUrls(String sinkName) throws Exception { sourceAttributes.put("source.mods.enable", "true"); sourceAttributes.put("user", "root"); - // Test forced-log mode, in open releases this might be "file" - sourceAttributes.put("source.realtime.mode", "forced-log"); + // Test batch mode + sourceAttributes.put("source.realtime.mode", "batch"); TSStatus status = client.createPipe( @@ -471,9 +457,9 @@ private void doTestUseNodeUrls(String sinkName) throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, "count databases", "count,", Collections.singleton("2,")); - // Test file mode + // Test batch mode after metadata transfer sourceAttributes.put("source.inclusion", "data"); - sourceAttributes.replace("source.realtime.mode", "file"); + sourceAttributes.replace("source.realtime.mode", "batch"); status = client.createPipe( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java index fce4361163fec..730416cfd5188 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSourceIT.java @@ -810,7 +810,7 @@ public void testExtractorTimeRangeMatch() throws Exception { sourceAttributes.put("source.inclusion", "data"); sourceAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00"); sourceAttributes.put("source.end-time", "1970-01-01T08:00:04+08:00"); - sourceAttributes.put("source.realtime.mode", "stream"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("user", "root"); final TSStatus status = diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java index 0c2f3bcb312a3..b2a98fe96f5e6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java @@ -140,7 +140,7 @@ public void testRevertParameterOrder() { String.format( "create pipe p1" + " with source (" - + "'source.realtime.mode'='hybrid'," + + "'source.realtime.mode'='batch'," + "'source.history.enable'='false') " + " with sink (" + "'sink.batch.enable'='false', " @@ -174,7 +174,7 @@ public void testRevertStageOrder() throws Exception { + "'sink.ip'='%s'," + "'sink'='iotdb-thrift-sink') " + " with source (" - + "'source.realtime.mode'='hybrid'," + + "'source.realtime.mode'='batch'," + "'source.history.enable'='false')", receiverIp, receiverPort)); fail(); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java index 6953a5a23f4a9..479956b4b8e58 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java @@ -33,6 +33,7 @@ import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -85,6 +86,7 @@ public void setUp() { } @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLivingAutoConflict() throws Exception { // Double living is two clusters each with a pipe connecting to the other. final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); @@ -239,6 +241,7 @@ public void testDoubleLivingAutoConflict() throws Exception { } @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLivingAutoConflictTemplate() throws Exception { final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java index 15ae9fca4c539..2236278919eb3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java @@ -217,17 +217,12 @@ private void testMachineDowntime(String sink) { @Test public void testWithAllParametersInLogMode() throws Exception { - testWithAllParameters("log"); + testWithAllParameters("batch"); } @Test public void testWithAllParametersInFileMode() throws Exception { - testWithAllParameters("file"); - } - - @Test - public void testWithAllParametersInHybridMode() throws Exception { - testWithAllParameters("hybrid"); + testWithAllParameters("batch"); } public void testWithAllParameters(final String realtimeMode) throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java index 7d0e7f71c0a87..f7a28936fe2fd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java @@ -58,7 +58,7 @@ public void testBasicCreatePipeIfNotExists() throws Exception { // Create pipe String sql = String.format( - "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='stream') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", + "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='batch') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -80,8 +80,7 @@ public void testBasicCreatePipeIfNotExists() throws Exception { // Check configurations Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -117,8 +116,7 @@ public void testBasicCreatePipeIfNotExists() throws Exception { // Check configurations Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); - Assert.assertTrue( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + Assert.assertTrue(showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertTrue( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( @@ -200,7 +198,7 @@ public void testBasicAlterPipeIfExists() throws Exception { // Create pipe sql = String.format( - "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='stream') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", + "create pipe If Not Exists a2b with source ('source'='iotdb-source', 'source.pattern'='root.test1', 'source.realtime.mode'='batch') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { @@ -234,7 +232,7 @@ public void testBasicAlterPipeIfExists() throws Exception { Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source=iotdb-source")); Assert.assertFalse(showPipeResult.get(0).pipeExtractor.contains("source.pattern=root.test1")); Assert.assertFalse( - showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=stream")); + showPipeResult.get(0).pipeExtractor.contains("source.realtime.mode=batch")); Assert.assertFalse( showPipeResult.get(0).pipeProcessor.contains("processor=do-nothing-processor")); Assert.assertTrue( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java index fb8945471b77e..03ee29d8cb931 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java @@ -427,7 +427,6 @@ private void testIdempotent( sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java index 5424c9f87a7ef..5a30525d3943c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeSinkCompressionIT.java @@ -60,7 +60,6 @@ public class IoTDBPipeSinkCompressionIT extends AbstractPipeDualTreeModelAutoIT @Override @Before public void setUp() { - // Override to enable air-gap MultiEnvFactory.createEnv(2); senderEnv = MultiEnvFactory.getEnv(0); receiverEnv = MultiEnvFactory.getEnv(1); @@ -98,7 +97,7 @@ public void setUp() { @Test public void testCompression1() throws Exception { - doTest("iotdb-thrift-sink", "stream", true, "snappy"); + doTest("iotdb-thrift-sink", "batch", true, "snappy"); } @Test @@ -108,7 +107,7 @@ public void testCompression2() throws Exception { @Test public void testCompression3() throws Exception { - doTest("iotdb-thrift-sync-sink", "stream", false, "snappy, snappy"); + doTest("iotdb-thrift-sync-sink", "batch", false, "snappy, snappy"); } @Test @@ -116,26 +115,13 @@ public void testCompression4() throws Exception { doTest("iotdb-thrift-sync-sink", "batch", true, "gzip, zstd"); } - @Test - public void testCompression5() throws Exception { - doTest("iotdb-air-gap-sink", "stream", false, "lzma2, lz4"); - } - - @Test - public void testCompression6() throws Exception { - doTest("iotdb-air-gap-sink", "batch", true, "lzma2"); - } - private void doTest( String sinkType, String realtimeMode, boolean useBatchMode, String compressionTypes) throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = - sinkType.contains("air-gap") - ? receiverDataNode.getPipeAirGapReceiverPort() - : receiverDataNode.getPort(); + final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java index b221800337988..ac48b6d1f3c36 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeTypeConversionSemanticIT.java @@ -156,7 +156,7 @@ private void createPipe() { senderEnv, String.format( "create pipe type_conversion_semantic" - + " with source ('source'='iotdb-source','source.path'='%s.**','history.enable'='false','realtime.mode'='forced-log')" + + " with source ('source'='iotdb-source','source.path'='%s.**','history.enable'='false','realtime.mode'='batch')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s','batch.enable'='false','sink.format'='tablet')", DEVICE, receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()), @@ -168,7 +168,7 @@ private void createStreamPipe(final String device) { senderEnv, String.format( "create pipe aligned_type_conversion_semantic" - + " with source ('source'='iotdb-source','source.path'='%s.**','history.enable'='false','realtime.mode'='stream')" + + " with source ('source'='iotdb-source','source.path'='%s.**','history.enable'='false','realtime.mode'='batch')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('sink'='iotdb-thrift-sink','sink.node-urls'='%s')", device, receiverEnv.getDataNodeWrapperList().get(0).getIpAndPortString()), diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java index 76b008166067f..796c77d1ffde6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java @@ -29,6 +29,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -43,6 +44,7 @@ @Category({MultiClusterIT2DualTreeManual.class}) public class IoTDBPipeManualConflictIT extends AbstractPipeDualTreeModelManualIT { @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLivingTimeseries() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -152,6 +154,7 @@ public void testDoubleLivingTimeseries() throws Exception { } @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testDoubleLivingTemplate() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java index 9645ff718df70..0bcb744c7ccdc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java @@ -86,7 +86,6 @@ public void testConfigNodeLeaderChange() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -141,7 +140,6 @@ public void testSchemaRegionLeaderChange() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java index 053fdee0731ac..3408e6362ba29 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java @@ -56,7 +56,6 @@ public void testAutoRestartSchemaTask() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -125,7 +124,6 @@ public void testAutoRestartConfigTask() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java index faafc5bf91394..b13d0b951a9a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java @@ -64,7 +64,6 @@ public void testMultiSchemaRegion() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.inclusion.exclusion", ""); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java index efb2eeee14a4d..e9ac2311ea575 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java @@ -397,7 +397,7 @@ private void createDataPipe(String diff, boolean isTSFile) { + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", diff, - isTSFile ? "file" : "forced-log", + "batch", !isTSFile, isTSFile, receiverEnv.getIP(), diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java index 82398bd712cd5..92269ad5d48ea 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java @@ -238,7 +238,7 @@ private void createDataPipe() { String sql = String.format( "create pipe test" - + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='forced-log')" + + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='batch')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='tablet')", receiverEnv.getIP(), receiverEnv.getPort()); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java index 11edc8cf6becc..0325807618d0c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeWriteBackSinkIT.java @@ -60,7 +60,7 @@ public void testWriteBackSinkWithTargetDatabaseForTreeModel() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("source.inclusion", "data.insert"); - sourceAttributes.put("source.forwarding-pipe-requests", "false"); + sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("source.path", "root.source.**"); sourceAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java deleted file mode 100644 index 9430e0adbfa42..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeAggregateIT.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.single; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT1; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT1.class}) -public class IoTDBPipeAggregateIT extends AbstractPipeSingleIT { - @Test - @Ignore - public void testAggregator() throws Exception { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { - // Test the mixture of historical and realtime data - // Do not fail if the failure has nothing to do with pipe - // Because the failures will randomly generate due to resource limitation - TestUtils.executeNonQueries( - env, - Arrays.asList( - "create timeseries root.ln.wf01.wt01.temperature with datatype=FLOAT, encoding=RLE, compression=SNAPPY tags(tag1=v1, tag2=v2) attributes(attr1=v1, attr2=v2)", - "create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN, encoding=RLE, compression=SNAPPY", - "insert into root.ln.wf01.wt01(time, temperature, status) values (10000, 1, false)"), - null); - - final Map sourceAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map sinkAttributes = new HashMap<>(); - - sourceAttributes.put("pattern", "root.ln"); - - processorAttributes.put("processor", "aggregate-processor"); - processorAttributes.put("output.database", "root.testdb"); - processorAttributes.put( - "output.measurements", "Avg1, peak1, rms1, var1, skew1, kurt1, ff1, cf1, pf1"); - processorAttributes.put( - "operators", "avg, peak, rms, var, skew, kurt, ff, cf, pf, cE, max, min"); - processorAttributes.put("sliding.seconds", "60"); - - sinkAttributes.put("sink", "write-back-sink"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(sourceAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - // Test unsupported types - TestUtils.executeNonQueries( - env, - Arrays.asList( - "create timeSeries root.ln.wf01.wt01.boolean boolean", - "create timeSeries root.ln.wf01.wt01.date date", - "create timeSeries root.ln.wf01.wt01.text text", - "create timeSeries root.ln.wf01.wt01.string string", - "create timeSeries root.ln.wf01.wt01.blob blob", - "insert into root.ln.wf01.wt01(time, boolean, date, text, string, blob) values (20000, false, '2000-12-13', 'abc', 'def', X'f103')", - "flush"), - null); - - TestUtils.executeNonQueries( - env, - Arrays.asList( - "insert into root.ln.wf01.wt01(time, temperature, status) values (20000, 2, true)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (30000, 3, false)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (40000, 4, true)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (50000, 5, false)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (60000, 6, true)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (70000, 7, false)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (80000, 8, true)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (90000, 9, false)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (100000, 10, true)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (110000, 11, false)", - "insert into root.ln.wf01.wt01(time, temperature, status) values (120000, 12, false)", - "flush"), - null); - - // Test total number - TestUtils.assertDataEventuallyOnEnv( - env, - "select count(*) from root.testdb.** group by level=1", - "count(root.testdb.*.*.*.*),", - Collections.singleton("24,")); - - // Test manually renamed timeSeries count - TestUtils.assertDataEventuallyOnEnv( - env, - "select count(Avg1) from root.testdb.wf01.wt01.temperature", - "count(root.testdb.wf01.wt01.temperature.Avg1),", - Collections.singleton("2,")); - - // Test default renamed timeSeries count - TestUtils.assertDataEventuallyOnEnv( - env, - "select count(cE) from root.testdb.wf01.wt01.temperature", - "count(root.testdb.wf01.wt01.temperature.cE),", - Collections.singleton("2,")); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java deleted file mode 100644 index 9dfa3bc012c14..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.single; - -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.db.pipe.sink.protocol.opcua.client.ClientRunner; -import org.apache.iotdb.db.pipe.sink.protocol.opcua.client.IoTDBOpcUaClient; -import org.apache.iotdb.it.env.MultiEnvFactory; -import org.apache.iotdb.it.env.cluster.EnvUtils; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT1; -import org.apache.iotdb.pipe.api.exception.PipeException; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.eclipse.milo.opcua.sdk.client.OpcUaClient; -import org.eclipse.milo.opcua.sdk.client.api.identity.AnonymousProvider; -import org.eclipse.milo.opcua.sdk.client.api.identity.IdentityProvider; -import org.eclipse.milo.opcua.sdk.client.api.identity.UsernameProvider; -import org.eclipse.milo.opcua.stack.core.security.SecurityPolicy; -import org.eclipse.milo.opcua.stack.core.types.builtin.DataValue; -import org.eclipse.milo.opcua.stack.core.types.builtin.DateTime; -import org.eclipse.milo.opcua.stack.core.types.builtin.NodeId; -import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; -import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; -import org.eclipse.milo.opcua.stack.core.types.enumerated.TimestampsToReturn; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.io.File; -import java.net.ConnectException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; - -import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_OPC_UA_SECURITY_DIR_DEFAULT_VALUE; -import static org.apache.iotdb.db.pipe.sink.protocol.opcua.server.OpcUaNameSpace.timestampToUtc; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT1.class}) -public class IoTDBPipeOPCUAIT extends AbstractPipeSingleIT { - - @Before - public void setUp() { - MultiEnvFactory.createEnv(1); - env = MultiEnvFactory.getEnv(0); - env.getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true) - .setDatanodeMemoryProportion("3:3:1:1:1:0") - .setPipeMemoryManagementEnabled(false) - .setDataReplicationFactor(1) - .setSchemaReplicationFactor(1) - .setIsPipeEnableMemoryCheck(false) - .setPipeAutoSplitFullEnabled(false); - env.initClusterEnvironment(1, 1); - } - - @Test - public void testOPCUAServerSink() throws Exception { - int tcpPort = -1; - int httpsPort = -1; - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { - - TestUtils.executeNonQuery(env, "insert into root.db.d1(time, `1`) values (1, 1)", null); - - final Map sinkAttributes = new HashMap<>(); - - sinkAttributes.put("sink", "opc-ua-sink"); - sinkAttributes.put("model", "client-server"); - sinkAttributes.put("opcua.security-policy", "None"); - - OpcUaClient opcUaClient; - DataValue value; - while (true) { - final int[] ports = EnvUtils.searchAvailablePorts(); - tcpPort = ports[0]; - httpsPort = ports[1]; - sinkAttributes.put("opcua.tcp.port", Integer.toString(tcpPort)); - sinkAttributes.put("https.port", Integer.toString(httpsPort)); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .createPipe( - new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(Collections.singletonMap("user", "root")) - .setProcessorAttributes(Collections.emptyMap())) - .getCode()); - - try { - opcUaClient = - getOpcUaClient( - "opc.tcp://127.0.0.1:" + tcpPort + "/iotdb", SecurityPolicy.None, "root", "root"); - } catch (final PipeException e) { - if (e.getCause() instanceof ConnectException) { - continue; - } else { - throw e; - } - } - value = - opcUaClient - .readValue(0, TimestampsToReturn.Both, new NodeId(2, "root/db/d1/`1`")) - .get(); - Assert.assertEquals(new Variant(1.0), value.getValue()); - Assert.assertEquals(new DateTime(timestampToUtc(1)), value.getSourceTime()); - opcUaClient.disconnect().get(); - break; - } - - // Create the region first to avoid tsFile parsing - TestUtils.executeNonQueries( - env, - Arrays.asList( - "create aligned timeSeries root.db.`123`(value double, quality boolean, other int32)", - "create aligned timeSeries root.db.`1231`(value double, quality boolean, other int32)", - "create aligned timeSeries root.db.`1232`(value double, quality boolean, other int32)", - "insert into root.db.`123`(time, value, quality, other) values (0, 0, true, 1)", - "insert into root.db.`1231`(time, value, quality, other) values (0, 0, true, 1)", - "insert into root.db.`1232`(time, value, quality, other) values (0, 0, true, 1)"), - null); - - while (true) { - final int[] ports = EnvUtils.searchAvailablePorts(); - tcpPort = ports[0]; - httpsPort = ports[1]; - sinkAttributes.put("opcua.tcp.port", Integer.toString(tcpPort)); - sinkAttributes.put("https.port", Integer.toString(httpsPort)); - sinkAttributes.put("with-quality", "true"); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .alterPipe( - new TAlterPipeReq() - .setPipeName("testPipe") - .setIsReplaceAllConnectorAttributes(true) - .setConnectorAttributes(sinkAttributes) - .setProcessorAttributes(Collections.emptyMap()) - .setExtractorAttributes(Collections.emptyMap())) - .getCode()); - try { - opcUaClient = - getOpcUaClient( - "opc.tcp://127.0.0.1:" + tcpPort + "/iotdb", SecurityPolicy.None, "root", "root"); - } catch (final PipeException e) { - if (e.getCause() instanceof ConnectException) { - continue; - } else { - throw e; - } - } - break; - } - - // Test multiple regions - TestUtils.executeNonQueries( - env, - Arrays.asList( - "insert into root.db.`123`(time, value, quality, other) values (1, 1, false, 1)", - "insert into root.db.`1231`(time, value, quality, other) values (1, 1, false, 1)", - "insert into root.db.`1232`(time, value, quality, other) values (1, 1, false, 1)"), - null); - - long startTime = System.currentTimeMillis(); - while (true) { - try { - value = - opcUaClient - .readValue(0, TimestampsToReturn.Both, new NodeId(2, "root/db/`123`")) - .get(); - Assert.assertEquals(new Variant(1.0), value.getValue()); - Assert.assertEquals(StatusCode.BAD, value.getStatusCode()); - Assert.assertEquals(new DateTime(timestampToUtc(1)), value.getSourceTime()); - - value = - opcUaClient - .readValue(0, TimestampsToReturn.Both, new NodeId(2, "root/db/`1231`")) - .get(); - Assert.assertEquals(new Variant(1.0), value.getValue()); - Assert.assertEquals(StatusCode.BAD, value.getStatusCode()); - Assert.assertEquals(new DateTime(timestampToUtc(1)), value.getSourceTime()); - - value = - opcUaClient - .readValue(0, TimestampsToReturn.Both, new NodeId(2, "root/db/`1232`")) - .get(); - Assert.assertEquals(new Variant(1.0), value.getValue()); - Assert.assertEquals(StatusCode.BAD, value.getStatusCode()); - Assert.assertEquals(new DateTime(timestampToUtc(1)), value.getSourceTime()); - break; - } catch (final Throwable t) { - if (System.currentTimeMillis() - startTime > 10_000L) { - throw t; - } - } - } - - TestUtils.executeNonQuery( - env, "insert into root.db.`123`(time, quality) values (2, true)", null); - TestUtils.executeNonQuery(env, "insert into root.db.`123`(time, value) values (2, 2)", null); - - startTime = System.currentTimeMillis(); - while (true) { - try { - value = - opcUaClient - .readValue(0, TimestampsToReturn.Both, new NodeId(2, "root/db/`123`")) - .get(); - Assert.assertEquals(new DateTime(timestampToUtc(2)), value.getSourceTime()); - Assert.assertEquals(new Variant(2.0), value.getValue()); - Assert.assertEquals(StatusCode.UNCERTAIN, value.getStatusCode()); - break; - } catch (final Throwable t) { - if (System.currentTimeMillis() - startTime > 10_000L) { - throw t; - } - } - } - - opcUaClient.disconnect().get(); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); - - // Test reconstruction - sinkAttributes.put("password", "test"); - sinkAttributes.put("security-policy", "basic256sha256"); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .createPipe( - new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(Collections.emptyMap()) - .setProcessorAttributes(Collections.emptyMap())) - .getCode()); - - // Banned none, only allows basic256sha256 - final int finalTcpPort = tcpPort; - Assert.assertThrows( - PipeException.class, - () -> - getOpcUaClient( - "opc.tcp://127.0.0.1:" + finalTcpPort + "/iotdb", - SecurityPolicy.None, - "root", - "root")); - - // Test conflict - sinkAttributes.put("password", "conflict"); - try { - TestUtils.executeNonQuery( - env, - String.format( - "create pipe test1 ('sink'='opc-ua-sink', 'password'='conflict@pswd', 'tcp.port'='%s', 'https.port'='%s')", - tcpPort, httpsPort), - null); - Assert.fail(); - } catch (final Exception e) { - Assert.assertEquals( - String.format( - "org.apache.iotdb.jdbc.IoTDBSQLException: 1107: The existing server with tcp port %s and https port %s's password **** conflicts to the new password ****, reject reusing.", - tcpPort, httpsPort), - e.getMessage()); - } - } finally { - if (tcpPort >= 0) { - final String lockPath = EnvUtils.getLockFilePath(tcpPort); - if (!new File(lockPath).delete()) { - System.out.printf("Delete lock file %s failed%n", lockPath); - } - } - } - } - - @Test - public void testOPCUASinkInTableModel() throws Exception { - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { - - TableModelUtils.createDataBaseAndTable(env, "test", "test"); - TableModelUtils.insertData("test", "test", 0, 10, env); - - final Map sourceAttributes = new HashMap<>(); - final Map sinkAttributes = new HashMap<>(); - sourceAttributes.put("capture.table", "true"); - sourceAttributes.put("__system.sql-dialect", "table"); - sourceAttributes.put("user", "root"); - - sinkAttributes.put("sink", "opc-ua-sink"); - sinkAttributes.put("opcua.model", "client-server"); - - final int[] ports = EnvUtils.searchAvailablePorts(); - final int tcpPort = ports[0]; - final int httpsPort = ports[1]; - sinkAttributes.put("tcp.port", Integer.toString(tcpPort)); - sinkAttributes.put("https.port", Integer.toString(httpsPort)); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .createPipe( - new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(sourceAttributes) - .setProcessorAttributes(Collections.emptyMap())) - .getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); - - // Test reconstruction - sinkAttributes.put("password123456", "test"); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .createPipe( - new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(Collections.emptyMap()) - .setProcessorAttributes(Collections.emptyMap())) - .getCode()); - - // Test conflict - sinkAttributes.put("password123456", "conflict"); - Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), - client - .createPipe( - new TCreatePipeReq("testPipe", sinkAttributes) - .setExtractorAttributes(Collections.emptyMap()) - .setProcessorAttributes(Collections.emptyMap())) - .getCode()); - } - } - - private static OpcUaClient getOpcUaClient( - final String nodeUrl, - final SecurityPolicy policy, - final String userName, - final String password) { - final IoTDBOpcUaClient client; - - final IdentityProvider provider = - Objects.nonNull(userName) - ? new UsernameProvider(userName, password) - : new AnonymousProvider(); - - final String securityDir = - CONNECTOR_OPC_UA_SECURITY_DIR_DEFAULT_VALUE - + File.separatorChar - + UUID.nameUUIDFromBytes(nodeUrl.getBytes(TSFileConfig.STRING_CHARSET)); - - client = new IoTDBOpcUaClient(nodeUrl, policy, provider, false); - new ClientRunner(client, securityDir, password, userName, 10).run(); - return client.getClient(); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java index 45fa762b02bf0..fe06ea53b8158 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipePermissionIT.java @@ -186,38 +186,6 @@ public void testSinkPermission() { } catch (Exception e) { fail(e.getMessage()); } - - // Tree model - try (final Connection connection = env.getConnection(); - final Statement statement = connection.createStatement()) { - statement.execute("CREATE DATABASE root.test_sink"); - statement.execute( - "CREATE ALIGNED TIMESERIES root.test_sink.d1(s_int INT32,s_long INT64,s_float float,s_double double)"); - statement.execute( - "INSERT INTO root.test_sink.d1(time,s_int,s_long,s_float,s_double) VALUES (2025-01-01 03:50:14,1,1,1,1),(2025-01-01 03:50:30,12,14,16.24,18.5),(2025-01-01 03:51:45,22,24,26.24,28.5),(2025-01-01 03:51:59,32,34,36.46,38.6),(2025-01-01 03:52:00,10,14,16.46,18.6)"); - statement.execute("CREATE USER user_new 'paSs1234@56789'"); - // Do not grant SYSTEM privilege to avoid auto creation - statement.execute("GRANT write ON root.** TO USER user_new"); - statement.execute( - "create pipe user_sink_pipe with source ('pattern'='root.test_sink.d1') with processor ('processor'='aggregate-processor', 'output.database'='root.user_sink_db', 'operators'='avg') with sink ('sink'='write-back-sink','user'='user_new','password'='paSs1234@56789')"); - - final long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime <= 20_000L) { - try (final ResultSet set = statement.executeQuery("show pipe user_sink_pipe")) { - Assert.assertTrue(set.next()); - try { - Assert.assertEquals("0", set.getString(8)); - return; - } catch (final Throwable t) { - // Retry - } - } - } - Assert.fail(); - } catch (Exception e) { - e.printStackTrace(); - fail(); - } } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeSingleEnvDemoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeSingleEnvDemoIT.java index a8c2990c5f781..11f9c2099d973 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeSingleEnvDemoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeSingleEnvDemoIT.java @@ -45,7 +45,7 @@ public void testSingleEnv() throws Exception { final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); - extractorAttributes.put("extractor.realtime.mode", "log"); + extractorAttributes.put("extractor.realtime.mode", "batch"); connectorAttributes.put("connector", "iotdb-thrift-connector"); connectorAttributes.put("connector.ip", "127.0.0.1"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java index 3f2cab3450181..efe403a906f79 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/triple/IoTDBPipeForwardIT.java @@ -29,6 +29,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -42,6 +43,7 @@ @Category({MultiClusterIT3.class}) public class IoTDBPipeForwardIT extends AbstractPipeTripleManualIT { @Test + @Ignore("Requires forwarding pipe request filtering, which is disabled.") public void testForwardingPipeRequests() throws Exception { final DataNodeWrapper env2DataNode = env2.getDataNodeWrapper(0); final String env2Ip = env2DataNode.getIp(); diff --git a/iotdb-client/client-go b/iotdb-client/client-go index 2ea2655e090dc..dc64b1a7648d3 160000 --- a/iotdb-client/client-go +++ b/iotdb-client/client-go @@ -1 +1 @@ -Subproject commit 2ea2655e090dcefd12bf1a789a51c8df9a28fa24 +Subproject commit dc64b1a7648d3c505c10eed5419f422bb49f1def diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java index c7222e9a1bc97..66e41eb42549f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink; import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor; -import org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionAirGapSink; import org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionSink; import org.apache.iotdb.pipe.api.PipeConnector; @@ -41,9 +40,7 @@ protected void initConstructors() { pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(), IoTDBConfigRegionSink::new); - pluginConstructors.put( - BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(), - IoTDBConfigRegionAirGapSink::new); + pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); @@ -55,8 +52,7 @@ protected void initConstructors() { BuiltinPipePlugin.IOTDB_THRIFT_SYNC_SINK.getPipePluginName(), IoTDBConfigRegionSink::new); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(), IoTDBConfigRegionSink::new); - pluginConstructors.put( - BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBConfigRegionAirGapSink::new); + pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 2f3fdd5ff6472..5d4e3ff7776ce 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2483,4 +2483,7 @@ private DataNodePipeMessages() {} public static final String MESSAGE_TRANSFER_FILE_ARG_ERROR_RESULT_STATUS_ARG_E565D9FD = "Transfer file %s error, result status %s."; + public static final String EXCEPTION_THE_PARAMETER_ARG_CANNOT_BE_SET_TO_FALSE_111C3408 = + "The parameter %s cannot be set to false."; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index b2b67f4d4b51c..3c193d9dcdcec 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2322,4 +2322,7 @@ private DataNodePipeMessages() {} public static final String MESSAGE_TRANSFER_FILE_ARG_ERROR_RESULT_STATUS_ARG_E565D9FD = "传输文件 %s 出错，结果状态为 %s。"; + public static final String EXCEPTION_THE_PARAMETER_ARG_CANNOT_BE_SET_TO_FALSE_111C3408 = + "参数 %s 不能被设置为 false。"; + } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java index a6160e6b90088..5ecc33596292b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java @@ -21,18 +21,10 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.throwing.ThrowingExceptionProcessor; import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeProcessorConstructor; import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper; -import org.apache.iotdb.db.pipe.processor.aggregate.AggregateProcessor; -import org.apache.iotdb.db.pipe.processor.aggregate.operator.processor.StandardStatisticsOperatorProcessor; -import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.TumblingWindowingProcessor; -import org.apache.iotdb.db.pipe.processor.downsampling.changing.ChangingValueSamplingProcessor; -import org.apache.iotdb.db.pipe.processor.downsampling.sdt.SwingingDoorTrendingSamplingProcessor; -import org.apache.iotdb.db.pipe.processor.downsampling.tumbling.TumblingTimeSamplingProcessor; import org.apache.iotdb.db.pipe.processor.iotconsensusv2.IoTConsensusV2Processor; import org.apache.iotdb.db.pipe.processor.schemachange.RenameDatabaseProcessor; -import org.apache.iotdb.db.pipe.processor.twostage.plugin.TwoStageCountProcessor; class PipeDataRegionProcessorConstructor extends PipeProcessorConstructor { @@ -44,28 +36,7 @@ class PipeDataRegionProcessorConstructor extends PipeProcessorConstructor { protected void initConstructors() { pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_PROCESSOR.getPipePluginName(), DoNothingProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.TUMBLING_TIME_SAMPLING_PROCESSOR.getPipePluginName(), - TumblingTimeSamplingProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.SDT_SAMPLING_PROCESSOR.getPipePluginName(), - SwingingDoorTrendingSamplingProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName(), - ChangingValueSamplingProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.THROWING_EXCEPTION_PROCESSOR.getPipePluginName(), - ThrowingExceptionProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.AGGREGATE_PROCESSOR.getPipePluginName(), AggregateProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.STANDARD_STATISTICS_PROCESSOR.getPipePluginName(), - StandardStatisticsOperatorProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.TUMBLING_WINDOWING_PROCESSOR.getPipePluginName(), - TumblingWindowingProcessor::new); - pluginConstructors.put( - BuiltinPipePlugin.COUNT_POINT_PROCESSOR.getPipePluginName(), TwoStageCountProcessor::new); + pluginConstructors.put( BuiltinPipePlugin.IOT_CONSENSUS_V2_PROCESSOR.getPipePluginName(), IoTConsensusV2Processor::new); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java index c8e87890afd69..3443af6c2064a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java @@ -23,11 +23,8 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink; import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor; import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper; -import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink; import org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.IoTConsensusV2AsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink; -import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink; -import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink; import org.apache.iotdb.db.pipe.sink.protocol.websocket.WebSocketSink; @@ -63,13 +60,9 @@ protected void initConstructors() { pluginConstructors.put( BuiltinPipePlugin.IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName(), IoTDBLegacyPipeSink::new); - pluginConstructors.put( - BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(), - IoTDBDataRegionAirGapSink::new); + pluginConstructors.put( BuiltinPipePlugin.WEBSOCKET_CONNECTOR.getPipePluginName(), WebSocketSink::new); - pluginConstructors.put(BuiltinPipePlugin.OPC_UA_CONNECTOR.getPipePluginName(), OpcUaSink::new); - pluginConstructors.put(BuiltinPipePlugin.OPC_DA_CONNECTOR.getPipePluginName(), OpcDaSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( @@ -86,12 +79,9 @@ protected void initConstructors() { IoTDBDataRegionAsyncSink::new); pluginConstructors.put( BuiltinPipePlugin.IOTDB_LEGACY_PIPE_SINK.getPipePluginName(), IoTDBLegacyPipeSink::new); - pluginConstructors.put( - BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBDataRegionAirGapSink::new); + pluginConstructors.put( BuiltinPipePlugin.WEBSOCKET_SINK.getPipePluginName(), WebSocketSink::new); - pluginConstructors.put(BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName(), OpcUaSink::new); - pluginConstructors.put(BuiltinPipePlugin.OPC_DA_SINK.getPipePluginName(), OpcDaSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java index 160ecf54c0158..bdec09bc07175 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink; import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor; -import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink; import org.apache.iotdb.pipe.api.PipeConnector; @@ -41,9 +40,7 @@ protected void initConstructors() { pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(), IoTDBSchemaRegionSink::new); - pluginConstructors.put( - BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(), - IoTDBSchemaRegionAirGapSink::new); + pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); @@ -55,8 +52,7 @@ protected void initConstructors() { BuiltinPipePlugin.IOTDB_THRIFT_SYNC_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new); - pluginConstructors.put( - BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBSchemaRegionAirGapSink::new); + pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index 36261c42fd448..37dd094896877 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.source.IoTDBSource; @@ -51,6 +52,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.iotdb.pipe.api.exception.PipePasswordCheckException; import org.apache.iotdb.rpc.TSStatusCode; @@ -140,6 +142,21 @@ public class IoTDBDataRegionSource extends IoTDBSource { public void validate(final PipeParameterValidator validator) throws Exception { super.validate(validator); + final boolean forwardingPipeRequests = + validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), + PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + if (!forwardingPipeRequests) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_THE_PARAMETER_ARG_CANNOT_BE_SET_TO_FALSE_111C3408, + PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY)); + } + final Pair insertionDeletionListeningOptionPair = DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair( validator.getParameters()); @@ -190,32 +207,6 @@ public void validate(final PipeParameterValidator validator) throws Exception { Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY), EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)); - // Validate source.realtime.mode - if (validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY), - EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE) - || validator - .getParameters() - .hasAnyAttributes( - SOURCE_START_TIME_KEY, - EXTRACTOR_START_TIME_KEY, - SOURCE_END_TIME_KEY, - EXTRACTOR_END_TIME_KEY)) { - validator.validateAttributeValueRange( - validator.getParameters().hasAttribute(EXTRACTOR_REALTIME_MODE_KEY) - ? EXTRACTOR_REALTIME_MODE_KEY - : SOURCE_REALTIME_MODE_KEY, - true, - EXTRACTOR_REALTIME_MODE_FILE_VALUE, - EXTRACTOR_REALTIME_MODE_HYBRID_VALUE, - EXTRACTOR_REALTIME_MODE_LOG_VALUE, - EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE, - EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE, - EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE); - } - checkInvalidParameters(validator); constructHistoricalSource(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java index 62136877fc1b0..9e03d88e003af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java @@ -120,7 +120,7 @@ public abstract class PipeRealtimeDataRegionSource implements PipeExtractor { private final AtomicReference> dataRegionTimePartitionIdBound = new AtomicReference<>(); - protected boolean isForwardingPipeRequests; + protected boolean isForwardingPipeRequests = true; private boolean shouldTransferModFile; // Whether to transfer mods @@ -262,22 +262,7 @@ public void customize( ? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) : TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - 1; - final boolean isDoubleLiving = - parameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, - PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); - if (isDoubleLiving) { - isForwardingPipeRequests = false; - } else { - isForwardingPipeRequests = - parameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, - PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), - PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); - } + isForwardingPipeRequests = true; if (parameters.hasAnyAttributes(EXTRACTOR_MODS_KEY, SOURCE_MODS_KEY)) { shouldTransferModFile = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java index 9545d71d96569..6349f442f398d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.sink; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; @@ -72,17 +71,7 @@ public void testCalculateSinkSubtaskNumForDataRegionSink() { } @Test - public void testCalculateSinkSubtaskNumUsesSingleThreadDefaultSinkAndSchemaRegionLimit() { - final Map singleThreadAttributes = new HashMap<>(); - singleThreadAttributes.put( - PipeSinkConstant.CONNECTOR_SERIALIZE_BY_REGION_KEY, Boolean.FALSE.toString()); - singleThreadAttributes.put( - PipeSinkConstant.CONNECTOR_KEY, BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName()); - Assert.assertEquals( - 1, - PipeSinkSubtaskManager.calculateSinkSubtaskNum( - new PipeParameters(singleThreadAttributes), -1)); - + public void testCalculateSinkSubtaskNumUsesSchemaRegionLimit() { final Map schemaRegionAttributes = new HashMap<>(); schemaRegionAttributes.put( PipeSinkConstant.CONNECTOR_SERIALIZE_BY_REGION_KEY, Boolean.FALSE.toString()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java index fa18695ef1f1b..2df30b9b7279c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java @@ -27,7 +27,6 @@ import org.apache.iotdb.db.pipe.event.common.statement.PipeStatementInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink; -import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink; import org.apache.iotdb.db.pipe.sink.protocol.websocket.WebSocketConnectorServer; @@ -47,7 +46,6 @@ import org.mockito.Mockito; import java.lang.reflect.Field; -import java.security.SecureRandom; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -193,97 +191,6 @@ public void testWebSocketSinkDropDoesNotRequeueDroppedPipeEvents() { } } - @Test - public void testOpcUaSink() { - final List schemaList = - Arrays.asList( - new MeasurementSchema("s1", TSDataType.INT64), - new MeasurementSchema("s2", TSDataType.INT64)); - - final Tablet tablet = new Tablet("root.db.d1.vector6", schemaList, 100); - - long timestamp = System.currentTimeMillis(); - for (long row = 0; row < 100; row++) { - final int rowSize = tablet.getRowSize(); - tablet.addTimestamp(rowSize, timestamp); - for (int i = 0; i < 2; i++) { - tablet.addValue( - schemaList.get(i).getMeasurementName(), rowSize, new SecureRandom().nextLong()); - } - timestamp++; - } - - final List opcSchemaList = - Arrays.asList( - new MeasurementSchema("value1", TSDataType.INT64), - new MeasurementSchema("quality1", TSDataType.BOOLEAN)); - final Tablet qualityTablet = new Tablet("root.db.d1.vector6.s3", opcSchemaList, 100); - - timestamp = System.currentTimeMillis(); - for (long row = 0; row < 100; row++) { - final int rowSize = qualityTablet.getRowSize(); - qualityTablet.addTimestamp(rowSize, timestamp); - qualityTablet.addValue( - opcSchemaList.get(0).getMeasurementName(), rowSize, new SecureRandom().nextLong()); - qualityTablet.addValue(opcSchemaList.get(1).getMeasurementName(), rowSize, true); - timestamp++; - } - - try (final OpcUaSink qualityOPC = new OpcUaSink(); - final OpcUaSink normalOPC = new OpcUaSink()) { - final PipeTaskRuntimeConfiguration configuration = - new PipeTaskRuntimeConfiguration(new PipeTaskSinkRuntimeEnvironment("temp", 0, 1)); - qualityOPC.customize( - new PipeParameters( - new HashMap() { - { - put( - PipeSinkConstant.CONNECTOR_KEY, - BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName()); - put(PipeSinkConstant.CONNECTOR_OPC_UA_WITH_QUALITY_KEY, "true"); - put(PipeSinkConstant.CONNECTOR_OPC_UA_VALUE_NAME_KEY, "value1"); - put(PipeSinkConstant.CONNECTOR_OPC_UA_QUALITY_NAME_KEY, "quality1"); - } - }), - configuration); - normalOPC.customize( - new PipeParameters( - new HashMap() { - { - put( - PipeSinkConstant.CONNECTOR_KEY, - BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName()); - } - }), - configuration); - final PipeRawTabletInsertionEvent event = - new PipeRawTabletInsertionEvent( - false, "root.db", "db", "root.db", tablet, false, "pipe", 0L, null, null, false); - event.increaseReferenceCount(""); - normalOPC.transfer(event); - // Shall not throw - qualityOPC.transfer(event); - event.decreaseReferenceCount("", false); - - qualityOPC.transfer( - new PipeRawTabletInsertionEvent( - false, - "root.db", - "db", - "root.db", - qualityTablet, - false, - "pipe", - 0L, - null, - null, - false)); - - } catch (Exception e) { - Assert.fail(); - } - } - @Test public void testWriteBackSinkTargetDatabaseValidation() throws Exception { assertWriteBackSinkTargetDatabaseValid("target"); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java index d49ef82ce0550..17b0d386026ed 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.pipe.sink.protocol.airgap; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration; import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSinkRuntimeEnvironment; @@ -106,9 +105,7 @@ public void testTransferTsFileBatchOverAirGap() throws Exception { private PipeParameters buildParameters(final boolean useTsFileBatch) { final Map attributes = new HashMap<>(); - attributes.put( - PipeSinkConstant.CONNECTOR_KEY, - BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); + attributes.put(PipeSinkConstant.CONNECTOR_KEY, "iotdb-air-gap-connector"); attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_NODE_URLS_KEY, "127.0.0.1:6668"); attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_BATCH_DELAY_MS_KEY, "200"); attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY, "1048576"); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java index 76b45d1e79895..eb23ad70133c9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java @@ -19,27 +19,16 @@ package org.apache.iotdb.commons.pipe.agent.plugin.builtin; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.AggregateProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.StandardStatisticsProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.TumblingWindowingProcessor; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.ChangingValueSamplingProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.SwingingDoorTrendingSamplingProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.TumblingTimeSamplingProcessor; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.iotconsensusv2.IoTConsensusV2Processor; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.schemachange.RenameDatabaseProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.throwing.ThrowingExceptionProcessor; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.twostage.TwoStageCountProcessor; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.airgap.IoTDBAirGapSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.consensus.IoTConsensusV2AsyncSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBLegacyPipeSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftAsyncSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.writeback.WriteBackSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.source.donothing.DoNothingSource; @@ -61,18 +50,8 @@ public enum BuiltinPipePlugin { // processors DO_NOTHING_PROCESSOR("do-nothing-processor", DoNothingProcessor.class), - TUMBLING_TIME_SAMPLING_PROCESSOR( - "tumbling-time-sampling-processor", TumblingTimeSamplingProcessor.class), - SDT_SAMPLING_PROCESSOR("sdt-sampling-processor", SwingingDoorTrendingSamplingProcessor.class), - CHANGING_VALUE_SAMPLING_PROCESSOR( - "changing-value-sampling-processor", ChangingValueSamplingProcessor.class), - THROWING_EXCEPTION_PROCESSOR("throwing-exception-processor", ThrowingExceptionProcessor.class), - AGGREGATE_PROCESSOR("aggregate-processor", AggregateProcessor.class), - COUNT_POINT_PROCESSOR("count-point-processor", TwoStageCountProcessor.class), // Hidden-processors, which are plugins of the processors - STANDARD_STATISTICS_PROCESSOR("standard-statistics-processor", StandardStatisticsProcessor.class), - TUMBLING_WINDOWING_PROCESSOR("tumbling-windowing-processor", TumblingWindowingProcessor.class), IOT_CONSENSUS_V2_PROCESSOR("iot-consensus-v2-processor", IoTConsensusV2Processor.class), // Legacy alias for stale PipeMeta written before the PipeConsensus -> IoTConsensusV2 rename. PIPE_CONSENSUS_PROCESSOR("pipe-consensus-processor", IoTConsensusV2Processor.class), @@ -85,15 +64,12 @@ public enum BuiltinPipePlugin { IOTDB_THRIFT_SYNC_CONNECTOR("iotdb-thrift-sync-connector", IoTDBThriftSyncSink.class), IOTDB_THRIFT_ASYNC_CONNECTOR("iotdb-thrift-async-connector", IoTDBThriftAsyncSink.class), IOTDB_LEGACY_PIPE_CONNECTOR("iotdb-legacy-pipe-connector", IoTDBLegacyPipeSink.class), - IOTDB_AIR_GAP_CONNECTOR("iotdb-air-gap-connector", IoTDBAirGapSink.class), IOT_CONSENSUS_V2_ASYNC_CONNECTOR( "iot-consensus-v2-async-connector", IoTConsensusV2AsyncSink.class), // Legacy alias for stale PipeMeta written before the PipeConsensus -> IoTConsensusV2 rename. PIPE_CONSENSUS_ASYNC_CONNECTOR("pipe-consensus-async-connector", IoTConsensusV2AsyncSink.class), WEBSOCKET_CONNECTOR("websocket-connector", WebSocketSink.class), - OPC_UA_CONNECTOR("opc-ua-connector", OpcUaSink.class), - OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class), WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class), DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class), @@ -102,10 +78,7 @@ public enum BuiltinPipePlugin { IOTDB_THRIFT_SYNC_SINK("iotdb-thrift-sync-sink", IoTDBThriftSyncSink.class), IOTDB_THRIFT_ASYNC_SINK("iotdb-thrift-async-sink", IoTDBThriftAsyncSink.class), IOTDB_LEGACY_PIPE_SINK("iotdb-legacy-pipe-sink", IoTDBLegacyPipeSink.class), - IOTDB_AIR_GAP_SINK("iotdb-air-gap-sink", IoTDBAirGapSink.class), WEBSOCKET_SINK("websocket-sink", WebSocketSink.class), - OPC_UA_SINK("opc-ua-sink", OpcUaSink.class), - OPC_DA_SINK("opc-da-sink", OpcDaSink.class), WRITE_BACK_SINK("write-back-sink", WriteBackSink.class), SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class), IOT_CONSENSUS_V2_ASYNC_SINK("iot-consensus-v2-async-sink", IoTConsensusV2AsyncSink.class), @@ -155,14 +128,6 @@ public String getClassName() { // Sources DO_NOTHING_SOURCE.getPipePluginName().toUpperCase(), // Processors - TUMBLING_TIME_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(), - SDT_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(), - CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(), - THROWING_EXCEPTION_PROCESSOR.getPipePluginName().toUpperCase(), - AGGREGATE_PROCESSOR.getPipePluginName().toUpperCase(), - COUNT_POINT_PROCESSOR.getPipePluginName().toUpperCase(), - STANDARD_STATISTICS_PROCESSOR.getPipePluginName().toUpperCase(), - TUMBLING_WINDOWING_PROCESSOR.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_PROCESSOR.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_PROCESSOR.getPipePluginName().toUpperCase(), RENAME_DATABASE_PROCESSOR.getPipePluginName().toUpperCase(), @@ -173,10 +138,7 @@ public String getClassName() { IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName().toUpperCase(), IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName().toUpperCase(), - IOTDB_AIR_GAP_CONNECTOR.getPipePluginName().toUpperCase(), WEBSOCKET_CONNECTOR.getPipePluginName().toUpperCase(), - OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(), - OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(), WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), @@ -185,8 +147,6 @@ public String getClassName() { IOTDB_THRIFT_ASYNC_SINK.getPipePluginName().toUpperCase(), IOTDB_LEGACY_PIPE_SINK.getPipePluginName().toUpperCase(), WEBSOCKET_SINK.getPipePluginName().toUpperCase(), - OPC_UA_SINK.getPipePluginName().toUpperCase(), - OPC_DA_SINK.getPipePluginName().toUpperCase(), SUBSCRIPTION_SINK.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_ASYNC_SINK.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_ASYNC_SINK.getPipePluginName().toUpperCase()))); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java index 39be064f09a41..bdb28766ae65e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java @@ -58,13 +58,7 @@ public class PipeSinkConstant { public static final String SINK_IOTDB_PARALLEL_TASKS_KEY = "sink.parallel.tasks"; public static final int CONNECTOR_IOTDB_PARALLEL_TASKS_DEFAULT_VALUE = PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(); - public static final Set SINGLE_THREAD_DEFAULT_SINK = - new HashSet<>( - Arrays.asList( - BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName(), - BuiltinPipePlugin.OPC_UA_CONNECTOR.getPipePluginName(), - BuiltinPipePlugin.OPC_DA_SINK.getPipePluginName(), - BuiltinPipePlugin.OPC_DA_CONNECTOR.getPipePluginName())); + public static final Set SINGLE_THREAD_DEFAULT_SINK = Collections.emptySet(); public static final String CONNECTOR_REALTIME_FIRST_KEY = "connector.realtime-first"; public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java index c2dd6855d010a..b5faf484212f9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java @@ -69,7 +69,7 @@ public abstract class IoTDBSource implements PipeExtractor { protected int regionId; protected PipeTaskMeta pipeTaskMeta; - protected boolean isForwardingPipeRequests; + protected boolean isForwardingPipeRequests = true; // The value is always true after the first start even the extractor is closed protected final AtomicBoolean hasBeenStarted = new AtomicBoolean(false); @@ -149,22 +149,7 @@ public void customize( pipeTaskMeta = ((PipeTaskSourceRuntimeEnvironment) environment).getPipeTaskMeta(); } - final boolean isDoubleLiving = - parameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, - PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); - if (isDoubleLiving) { - isForwardingPipeRequests = false; - } else { - isForwardingPipeRequests = - parameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, - PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), - PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); - } + isForwardingPipeRequests = true; userId = parameters.getStringOrDefault(