diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index 31cd0241f69a3..89d820932b79c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -113,6 +113,7 @@ public void testManageDatabase() { statement.execute("alter database if exists test1 set properties ttl='INF'"); statement.execute("alter database test set properties ttl=default"); + statement.execute("alter database test set properties need_last_cache=false"); String[] databaseNames = new String[] {"test"}; String[] TTLs = new String[] {"INF"}; @@ -168,6 +169,7 @@ public void testManageDatabase() { assertTrue(resultSet.getInt(7) >= defaultSchemaRegionGroupNum[cnt]); assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(8)); assertTrue(resultSet.getInt(9) >= defaultDataRegionGroupNum[cnt]); + assertFalse(resultSet.getBoolean(10)); cnt++; } assertEquals(databaseNames.length, cnt); @@ -318,6 +320,37 @@ public void testShowCreateDatabase() throws SQLException { } } + @Test + public void testNeedLastCacheDatabaseProperty() throws SQLException { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database need_cache_false with (need_last_cache=false)"); + statement.execute("create database need_cache_default"); + + assertDatabaseNeedLastCache(statement, "need_cache_false", false); + assertDatabaseNeedLastCache(statement, "need_cache_default", true); + + statement.execute("alter database need_cache_false set properties need_last_cache=true"); + assertDatabaseNeedLastCache(statement, "need_cache_false", true); + + statement.execute("alter database need_cache_false set properties need_last_cache=false"); + assertDatabaseNeedLastCache(statement, "need_cache_false", false); + + statement.execute("alter database need_cache_false set properties need_last_cache=default"); + assertDatabaseNeedLastCache(statement, "need_cache_false", true); + + try { + statement.execute("create database need_cache_invalid with (need_last_cache=1)"); + fail("non-boolean need_last_cache should be rejected"); + } catch (final SQLException e) { + assertEquals( + "701: need_last_cache value must be a BooleanLiteral, but now is LongLiteral, value: 1", + e.getMessage()); + } + } + } + @Test public void testShowCreatePipe() throws SQLException { try (final Connection connection = @@ -343,6 +376,30 @@ public void testShowCreateInformationSchemaDatabase() throws SQLException { } } + private static void assertDatabaseNeedLastCache( + final Statement statement, final String database, final boolean expected) + throws SQLException { + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { + boolean found = false; + while (resultSet.next()) { + if (!database.equals(resultSet.getString("Database"))) { + continue; + } + found = true; + assertEquals(expected, resultSet.getBoolean("NeedLastCache")); + } + assertTrue(found); + } + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select database, need_last_cache from information_schema.databases where database = '" + + database + + "'"), + "database,need_last_cache,", + Collections.singleton(database + "," + expected + ",")); + } + private static void assertShowCreateSystemDatabaseFails( final Statement statement, final String database) throws SQLException { try { @@ -500,7 +557,8 @@ public void testInformationSchema() throws SQLException { "schema_region_group_num,INT32,ATTRIBUTE,", "max_schema_region_group_num,INT32,ATTRIBUTE,", "data_region_group_num,INT32,ATTRIBUTE,", - "max_data_region_group_num,INT32,ATTRIBUTE,"))); + "max_data_region_group_num,INT32,ATTRIBUTE,", + "need_last_cache,BOOLEAN,ATTRIBUTE,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc tables"), "ColumnName,DataType,Category,", @@ -511,7 +569,8 @@ public void testInformationSchema() throws SQLException { "ttl(ms),STRING,ATTRIBUTE,", "status,STRING,ATTRIBUTE,", "comment,STRING,ATTRIBUTE,", - "table_type,STRING,ATTRIBUTE,"))); + "table_type,STRING,ATTRIBUTE,", + "need_last_cache,BOOLEAN,ATTRIBUTE,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc columns"), "ColumnName,DataType,Category,", @@ -705,7 +764,7 @@ public void testInformationSchema() throws SQLException { statement.execute( "create table test.test (a tag, b attribute, c int32 comment 'turbine') comment 'test'"); statement.execute( - "CREATE VIEW test.view_table (tag1 STRING TAG,tag2 STRING TAG,s11 INT32 FIELD,s3 INT32 FIELD FROM s2) RESTRICT WITH (ttl=100) AS root.\"a\".**"); + "CREATE VIEW test.view_table (tag1 STRING TAG,tag2 STRING TAG,s11 INT32 FIELD,s3 INT32 FIELD FROM s2) RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"a\".**"); try (final ResultSet resultSet = statement.executeQuery("select * from databases")) { final ResultSetMetaData metaData = resultSet.getMetaData(); @@ -722,6 +781,7 @@ public void testInformationSchema() throws SQLException { for (int columnIndex = 3; columnIndex <= 9; columnIndex++) { assertNull(resultSet.getObject(columnIndex)); } + assertFalse(resultSet.getBoolean(10)); } else { assertEquals("test", resultSet.getString(1)); assertEquals("INF", resultSet.getString(2)); @@ -732,6 +792,7 @@ public void testInformationSchema() throws SQLException { assertTrue(resultSet.getInt(7) >= 1); assertEquals(0, resultSet.getInt(8)); assertTrue(resultSet.getInt(9) >= 2); + assertTrue(resultSet.getBoolean(10)); } cnt++; } @@ -739,32 +800,32 @@ public void testInformationSchema() throws SQLException { } TestUtils.assertResultSetEqual( statement.executeQuery("show devices from tables where status = 'USING'"), - "database,table_name,ttl(ms),status,comment,table_type,", + "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", new HashSet<>( Arrays.asList( - "information_schema,databases,INF,USING,null,SYSTEM VIEW,", - "information_schema,tables,INF,USING,null,SYSTEM VIEW,", - "information_schema,columns,INF,USING,null,SYSTEM VIEW,", - "information_schema,queries,INF,USING,null,SYSTEM VIEW,", - "information_schema,regions,INF,USING,null,SYSTEM VIEW,", - "information_schema,topics,INF,USING,null,SYSTEM VIEW,", - "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,", - "information_schema,pipes,INF,USING,null,SYSTEM VIEW,", - "information_schema,services,INF,USING,null,SYSTEM VIEW,", - "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,", - "information_schema,views,INF,USING,null,SYSTEM VIEW,", - "information_schema,functions,INF,USING,null,SYSTEM VIEW,", - "information_schema,configurations,INF,USING,null,SYSTEM VIEW,", - "information_schema,keywords,INF,USING,null,SYSTEM VIEW,", - "information_schema,nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,table_disk_usage,INF,USING,null,SYSTEM VIEW,", - "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,connections,INF,USING,null,SYSTEM VIEW,", - "information_schema,current_queries,INF,USING,null,SYSTEM VIEW,", - "information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,", - "test,test,INF,USING,test,BASE TABLE,", - "test,view_table,100,USING,null,VIEW FROM TREE,"))); + "information_schema,databases,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,tables,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,columns,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,queries,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,regions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,topics,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,pipes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,services,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,views,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,functions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,configurations,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,keywords,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,table_disk_usage,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,connections,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,current_queries,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,false,", + "test,test,INF,USING,test,BASE TABLE,true,", + "test,view_table,100,USING,null,VIEW FROM TREE,true,"))); TestUtils.assertResultSetEqual( statement.executeQuery("count devices from tables where status = 'USING'"), "count(devices),", @@ -811,7 +872,7 @@ public void testInformationSchema() throws SQLException { statement.executeQuery("select * from views"), "database,table_name,view_definition,", Collections.singleton( - "test,view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" INT32 FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"a\".**,")); + "test,view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" INT32 FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"a\".**,")); TestUtils.assertResultSetEqual( statement.executeQuery( @@ -870,12 +931,26 @@ public void testMixedDatabase() throws SQLException { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("create database root.test"); + statement.execute("create database root.test with NEED_LAST_CACHE=false"); + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS root.test")) { + assertTrue(resultSet.next()); + assertEquals("root.test", resultSet.getString("Database")); + assertFalse(resultSet.getBoolean("NeedLastCache")); + assertFalse(resultSet.next()); + } Assert.assertThrows( IoTDBSQLException.class, () -> statement.execute( "alter database root.test WITH MAX_SCHEMA_REGION_GROUP_NUM=2, MAX_DATA_REGION_GROUP_NUM=3")); + try { + statement.execute("alter database root.test WITH NEED_LAST_CACHE=true"); + fail("tree database need_last_cache alter should be rejected"); + } catch (final SQLException e) { + assertEquals( + "701: The tree model database does not support alter need last cache now.", + e.getMessage()); + } statement.execute("insert into root.test.d1 (s1) values(1)"); statement.execute("drop database root.test"); } @@ -994,8 +1069,9 @@ public void testDBAuth() throws SQLException { Collections.singleton("information_schema,INF,null,null,null,")); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.databases"), - "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,max_schema_region_group_num,data_region_group_num,max_data_region_group_num,", - Collections.singleton("information_schema,INF,null,null,null,null,null,null,null,")); + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,max_schema_region_group_num,data_region_group_num,max_data_region_group_num,need_last_cache,", + Collections.singleton( + "information_schema,INF,null,null,null,null,null,null,null,false,")); } try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index c3a72d3c23a59..183fd3781cc58 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -135,6 +135,7 @@ public void testManageTable() { String[] ttls = new String[] {"INF"}; String[] statuses = new String[] {"USING"}; String[] comments = new String[] {"test"}; + String[] needLastCaches = new String[] {"true"}; statement.execute("use test2"); @@ -154,6 +155,7 @@ public void testManageTable() { assertEquals(ttls[cnt], resultSet.getString(2)); assertEquals(statuses[cnt], resultSet.getString(3)); assertEquals(comments[cnt], resultSet.getString(4)); + assertEquals(needLastCaches[cnt], resultSet.getString(6)); cnt++; } assertEquals(tableNames.length, cnt); @@ -198,6 +200,9 @@ public void testManageTable() { statement.execute("comment on table test1.table1 is 'new_test'"); comments = new String[] {"new_test"}; + + statement.execute("alter table test1.table1 set properties need_last_cache=false"); + needLastCaches = new String[] {"false"}; // using SHOW tables from try (final ResultSet resultSet = statement.executeQuery("SHOW tables details from test1")) { int cnt = 0; @@ -211,6 +216,7 @@ public void testManageTable() { assertEquals(tableNames[cnt], resultSet.getString(1)); assertEquals(ttls[cnt], resultSet.getString(2)); assertEquals(comments[cnt], resultSet.getString(4)); + assertEquals(needLastCaches[cnt], resultSet.getString(6)); cnt++; } assertEquals(tableNames.length, cnt); @@ -314,7 +320,7 @@ public void testManageTable() { statement.executeQuery("show create table table2"), "Table,Create Table,", Collections.singleton( - "table2,CREATE TABLE \"table2\" (\"t1\" TIMESTAMP TIME,\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000),")); + "table2,CREATE TABLE \"table2\" (\"t1\" TIMESTAMP TIME,\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000, need_last_cache=true),")); try { statement.execute("alter table table2 add column speed DOUBLE FIELD"); @@ -478,7 +484,7 @@ public void testManageTable() { } // After - statement.execute("COMMENT ON COLUMN table2.region_id IS '重庆'"); + statement.execute("COMMENT ON COLUMN table2.region_id IS '閲嶅簡'"); statement.execute("COMMENT ON COLUMN table2.region_id IS NULL"); statement.execute("COMMENT ON COLUMN test2.table2.t1 IS 'recent'"); statement.execute("COMMENT ON COLUMN test2.table2.region_id IS ''"); @@ -626,13 +632,121 @@ public void testManageTable() { statement.executeQuery("show create table test100"), "Table,Create Table,", Collections.singleton( - "test100,CREATE TABLE \"test100\" (\"t1\" TIMESTAMP TIME) WITH (ttl='INF'),")); + "test100,CREATE TABLE \"test100\" (\"t1\" TIMESTAMP TIME) WITH (ttl='INF', need_last_cache=true),")); } catch (final SQLException e) { e.printStackTrace(); fail(e.getMessage()); } } + @Test + public void testNeedLastCacheTableAndViewProperty() throws Exception { + try (final Connection treeConnection = EnvFactory.getEnv().getConnection(); + final Statement treeStatement = treeConnection.createStatement()) { + treeStatement.execute("create database root.need_cache_view_source"); + treeStatement.execute("create timeseries root.need_cache_view_source.d1.s1 int32"); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("create database need_cache_db with (need_last_cache=false)"); + statement.execute("create database need_cache_default_db"); + statement.execute("use need_cache_db"); + + statement.execute( + "create table inherited_table(time timestamp time, device_id string tag, temperature float field)"); + statement.execute( + "create table override_table(time timestamp time, device_id string tag, temperature float field) with (need_last_cache=true)"); + statement.execute( + "create table explicit_false_table(time timestamp time, device_id string tag, temperature float field) with (need_last_cache=false)"); + + assertTableNeedLastCache(statement, "need_cache_db", "inherited_table", false); + assertTableNeedLastCache(statement, "need_cache_db", "override_table", true); + assertTableNeedLastCache(statement, "need_cache_db", "explicit_false_table", false); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show create table explicit_false_table"), + "Table,Create Table,", + Collections.singleton( + "explicit_false_table,CREATE TABLE \"explicit_false_table\" (\"time\" TIMESTAMP TIME,\"device_id\" STRING TAG,\"temperature\" FLOAT FIELD) WITH (ttl='INF', need_last_cache=false),")); + + statement.execute("alter table inherited_table set properties need_last_cache=true"); + assertTableNeedLastCache(statement, "need_cache_db", "inherited_table", true); + + statement.execute("alter table inherited_table set properties need_last_cache=default"); + assertTableNeedLastCache(statement, "need_cache_db", "inherited_table", false); + + statement.execute("alter table override_table set properties need_last_cache=false"); + assertTableNeedLastCache(statement, "need_cache_db", "override_table", false); + + statement.execute("alter table override_table set properties need_last_cache=true"); + assertTableNeedLastCache(statement, "need_cache_db", "override_table", true); + + statement.execute("use need_cache_default_db"); + statement.execute( + "create table default_reset_table(time timestamp time, device_id string tag, temperature float field) with (need_last_cache=false)"); + statement.execute("alter table default_reset_table set properties need_last_cache=default"); + assertTableNeedLastCache(statement, "need_cache_default_db", "default_reset_table", true); + + statement.execute("use need_cache_db"); + statement.execute( + "create view explicit_false_view (tag1 string tag, s1 int32 field) restrict with (ttl=100, need_last_cache=false) as root.need_cache_view_source.**"); + assertTableNeedLastCache(statement, "need_cache_db", "explicit_false_view", false); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show create view explicit_false_view"), + "View,Create View,", + Collections.singleton( + "explicit_false_view,CREATE VIEW \"explicit_false_view\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"s1\" INT32 FIELD) RESTRICT WITH (ttl=100, need_last_cache=false) AS root.\"need_cache_view_source\".**,")); + + statement.execute("alter view explicit_false_view set properties need_last_cache=true"); + assertTableNeedLastCache(statement, "need_cache_db", "explicit_false_view", true); + + statement.execute("alter view explicit_false_view set properties need_last_cache=false"); + assertTableNeedLastCache(statement, "need_cache_db", "explicit_false_view", false); + + statement.execute("alter view explicit_false_view set properties need_last_cache=default"); + assertTableNeedLastCache(statement, "need_cache_db", "explicit_false_view", false); + + TestUtils.assertResultSetEqual( + statement.executeQuery("show create view explicit_false_view"), + "View,Create View,", + Collections.singleton( + "explicit_false_view,CREATE VIEW \"explicit_false_view\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"s1\" INT32 FIELD) RESTRICT WITH (ttl=100, need_last_cache=false) AS root.\"need_cache_view_source\".**,")); + } + } + + private static void assertTableNeedLastCache( + final Statement statement, + final String database, + final String table, + final boolean expectedNeedLastCache) + throws SQLException { + try (final ResultSet resultSet = + statement.executeQuery("show tables details from " + database)) { + boolean found = false; + while (resultSet.next()) { + if (!table.equals(resultSet.getString("TableName"))) { + continue; + } + found = true; + assertEquals(expectedNeedLastCache, resultSet.getBoolean("NeedLastCache")); + } + assertTrue(found); + } + + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select database, table_name, need_last_cache from information_schema.tables where database = '" + + database + + "' and table_name = '" + + table + + "'"), + "database,table_name,need_last_cache,", + Collections.singleton(database + "," + table + "," + expectedNeedLastCache + ",")); + } + @Test public void testTableAuth() throws SQLException { try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -649,7 +763,7 @@ public void testTableAuth() throws SQLException { Assert.assertThrows(SQLException.class, () -> userStmt.execute("select * from db.test")); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.tables where database = 'db'"), - "database,table_name,ttl(ms),status,comment,table_type,", + "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", Collections.emptySet()); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.columns where database = 'db'"), @@ -775,10 +889,10 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { statement.execute("create database root.another"); - statement.execute("create database root.`重庆`.`1`.b"); - statement.execute("create timeSeries root.`重庆`.`1`.b.`2`.S1 int32"); - statement.execute("create timeSeries root.`重庆`.`1`.b.`2`.s2 string"); - statement.execute("create timeSeries root.`重庆`.`1`.b.S1 int32"); + statement.execute("create database root.`閲嶅簡`.`1`.b"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`2`.S1 int32"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`2`.s2 string"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.S1 int32"); } catch (SQLException e) { fail(e.getMessage()); } @@ -797,13 +911,13 @@ public void testTreeViewTable() throws Exception { "701: Cannot specify view pattern to match more than one tree database.", e.getMessage()); } - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"閲嶅簡\".\"1\".**"); statement.execute("drop view tree_table"); } try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("create timeSeries root.`重庆`.`1`.b.`1`.s1 int32"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`1`.s1 int32"); } catch (SQLException e) { fail(e.getMessage()); } @@ -814,7 +928,7 @@ public void testTreeViewTable() throws Exception { statement.execute("use tree_view_db"); try { - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { final Set result = @@ -828,13 +942,13 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("drop timeSeries root.`重庆`.`1`.b.`1`.s1"); + statement.execute("drop timeSeries root.`閲嶅簡`.`1`.b.`1`.s1"); statement.execute("create device template t1 (S1 boolean, s9 int32)"); - statement.execute("set schema template t1 to root.`重庆`.`1`.b.`1`"); - statement.execute("create timeSeries root.`重庆`.`1`.b.`2`.f.g.h.S1 int32"); + statement.execute("set schema template t1 to root.`閲嶅簡`.`1`.b.`1`"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.`2`.f.g.h.S1 int32"); // Put schema cache - statement.execute("select S1, s2 from root.`重庆`.`1`.b.`2`"); + statement.execute("select S1, s2 from root.`閲嶅簡`.`1`.b.`2`"); } catch (SQLException e) { fail(e.getMessage()); } @@ -845,7 +959,7 @@ public void testTreeViewTable() throws Exception { statement.execute("use tree_view_db"); try { - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -855,7 +969,7 @@ public void testTreeViewTable() throws Exception { try { statement.execute( - "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**"); + "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -866,7 +980,7 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("create timeSeries root.`重庆`.`1`.b.e.s1 int32"); + statement.execute("create timeSeries root.`閲嶅簡`.`1`.b.e.s1 int32"); } catch (SQLException e) { fail(e.getMessage()); } @@ -887,7 +1001,7 @@ public void testTreeViewTable() throws Exception { // Temporary try { statement.execute( - "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 boolean from S1) as root.\"重庆\".\"1\".**"); + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 boolean from S1) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -896,14 +1010,14 @@ public void testTreeViewTable() throws Exception { try { statement.execute( - "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2, s8 field) as root.\"重庆\".\"1\".**"); + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2, s8 field) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals("528: Measurements not found for s8, cannot auto detect", e.getMessage()); } statement.execute( - "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2) as root.\"重庆\".\"1\".**"); + "create or replace view tree_table (tag1 tag, tag2 tag, S1 int32 field, s3 from s2) as root.\"閲嶅簡\".\"1\".**"); // Cannot be written try { @@ -932,8 +1046,8 @@ public void testTreeViewTable() throws Exception { TestUtils.assertResultSetEqual( statement.executeQuery("show tables details"), - "TableName,TTL(ms),Status,Comment,TableType,", - Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,")); + "TableName,TTL(ms),Status,Comment,TableType,NeedLastCache,", + Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,true,")); TestUtils.assertResultSetEqual( statement.executeQuery("desc view_table"), @@ -971,7 +1085,7 @@ public void testTreeViewTable() throws Exception { final Statement statement = connection.createStatement()) { // Test create & replace + restrict statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.`重庆`.`1`.**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.`閲嶅簡`.`1`.**"); fail(); } catch (final SQLException e) { assertTrue( @@ -992,7 +1106,7 @@ public void testTreeViewTable() throws Exception { .getConnection("testUser", "testUser123456", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -1013,7 +1127,7 @@ public void testTreeViewTable() throws Exception { .getConnection("testUser", "testUser123456", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -1023,7 +1137,7 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("grant read_schema on root.`重庆`.** to user testUser"); + statement.execute("grant read_schema on root.`閲嶅簡`.** to user testUser"); } catch (final SQLException e) { fail(e.getMessage()); } @@ -1033,7 +1147,7 @@ public void testTreeViewTable() throws Exception { .getConnection("testUser", "testUser123456", BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict with (ttl=100) as root.\"閲嶅簡\".\"1\".**"); fail(); } catch (final SQLException e) { assertEquals( @@ -1043,7 +1157,7 @@ public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); final Statement statement = connection.createStatement()) { - statement.execute("grant read_data on root.`重庆`.** to user testUser"); + statement.execute("grant read_data on root.`閲嶅簡`.** to user testUser"); } catch (final SQLException e) { fail(e.getMessage()); } @@ -1053,7 +1167,7 @@ public void testTreeViewTable() throws Exception { final Statement statement = connection.createStatement()) { statement.execute("alter database tree_view_db set properties ttl=100"); statement.execute( - "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict as root.\"重庆\".\"1\".**"); + "create or replace view tree_view_db.view_table (tag1 tag, tag2 tag, s11 int32 field, s3 from s2) restrict as root.\"閲嶅簡\".\"1\".**"); TestUtils.assertResultSetEqual( statement.executeQuery("show tables from tree_view_db"), @@ -1077,14 +1191,14 @@ public void testTreeViewTable() throws Exception { statement.executeQuery("show create view view_table"), "View,Create View,", Collections.singleton( - "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".\"1\".**,")); + "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"閲嶅簡\".\"1\".**,")); // Can also use "show create table" TestUtils.assertResultSetEqual( statement.executeQuery("show create table view_table"), "View,Create View,", Collections.singleton( - "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".\"1\".**,")); + "view_table,CREATE VIEW \"view_table\" (\"time\" TIMESTAMP TIME,\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"閲嶅簡\".\"1\".**,")); statement.execute("create table a ()"); try { diff --git a/iotdb-client/client-cpp/examples/README.md b/iotdb-client/client-cpp/examples/README.md index 763ec693bee23..f45d54c99039d 100644 --- a/iotdb-client/client-cpp/examples/README.md +++ b/iotdb-client/client-cpp/examples/README.md @@ -149,9 +149,9 @@ iotdb_session.dll **Prerequisites on the target PC** - **64-bit Windows** (examples are built x64). -- **[Microsoft Visual C++ Redistributable for Visual Studio 2015–2022](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist)** +- **[Microsoft Visual C++ Redistributable for Visual Studio 2015–2022](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist)** (x64). The SDK and examples are built with **`/MD`**; the redistributable - supplies `vcruntime140.dll`, `msvcp140.dll`, etc. + supplies `vcruntime140.dll`, `msvcp140.dll`, etc. Installing this package is enough—you do **not** need Visual Studio or the IoTDB SDK on the target machine. diff --git a/iotdb-client/client-cpp/examples/README_zh.md b/iotdb-client/client-cpp/examples/README_zh.md index 4adc38a3fc73b..2a660b4b6e98a 100644 --- a/iotdb-client/client-cpp/examples/README_zh.md +++ b/iotdb-client/client-cpp/examples/README_zh.md @@ -146,9 +146,9 @@ iotdb_session.dll **目标机器前置条件** - **64 位 Windows**（示例为 x64 构建）。 -- 安装 **[Microsoft Visual C++ 2015–2022 可再发行组件包（x64）](https://learn.microsoft.com/zh-cn/cpp/windows/latest-supported-vc-redist)**。 +- 安装 **[Microsoft Visual C++ 2015–2022 可再发行组件包（x64）](https://learn.microsoft.com/zh-cn/cpp/windows/latest-supported-vc-redist)**。 SDK 与示例均使用 **`/MD`**（动态 CRT），该安装包提供 `vcruntime140.dll`、 - `msvcp140.dll` 等运行时。 + `msvcp140.dll` 等运行时。 **仅安装此 Redistributable 即可**，目标机 **不需要** Visual Studio，也 **不需要** IoTDB SDK 头文件或 Thrift/Boost。 diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index e96d84e892c8c..07a5324ffb3d8 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -156,6 +156,7 @@ keyWords | MODELS | MODIFY | NAN + | NEED_LAST_CACHE | NODEID | NODES | NONE diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 2f91b573aef0e..cf9cda53f9a1d 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -116,6 +116,7 @@ databaseAttributesClause databaseAttributeClause : databaseAttributeKey operator_eq INTEGER_LITERAL + | NEED_LAST_CACHE operator_eq boolean_literal ; databaseAttributeKey diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 3950de50ea477..b734da4497858 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -1235,6 +1235,10 @@ MAX_DATA_REGION_GROUP_NUM : M A X '_' D A T A '_' R E G I O N '_' G R O U P '_' N U M ; +NEED_LAST_CACHE + : N E E D '_' L A S T '_' C A C H E + ; + CURRENT_TIMESTAMP : C U R R E N T '_' T I M E S T A M P ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 6b0c82ff1f056..7201a921af546 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -40,6 +40,7 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; +import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; @@ -124,6 +125,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -154,6 +156,19 @@ public class ClusterSchemaManager { private static final String CONSENSUS_WRITE_ERROR = "Failed in the write API executing the consensus layer due to: "; + public static boolean isNeedLastCacheEnabled(final TDatabaseSchema databaseSchema) { + return !databaseSchema.isSetNeedLastCache() || databaseSchema.isNeedLastCache(); + } + + private static boolean needInvalidateLastCache( + final TDatabaseSchema before, final TDatabaseSchema after) { + final TDatabaseSchema mergedSchema = deepCopyDatabaseSchema(before); + if (after.isSetNeedLastCache()) { + mergedSchema.setNeedLastCache(after.isNeedLastCache()); + } + return isNeedLastCacheEnabled(before) && !isNeedLastCacheEnabled(mergedSchema); + } + public ClusterSchemaManager( final IManager configManager, final ClusterSchemaInfo clusterSchemaInfo, @@ -229,7 +244,10 @@ public TSStatus alterDatabase( TSStatus result; final TDatabaseSchema databaseSchema = databaseSchemaPlan.getSchema(); - if (!isDatabaseExist(databaseSchema.getName())) { + final TDatabaseSchema originalSchema; + try { + originalSchema = deepCopyDatabaseSchema(getDatabaseSchemaByName(databaseSchema.getName())); + } catch (final DatabaseNotExistsException e) { // Reject if Database doesn't exist result = new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode()); result.setMessage( @@ -266,11 +284,17 @@ public TSStatus alterDatabase( isGeneratedByPipe ? new PipeEnrichedPlan(databaseSchemaPlan) : databaseSchemaPlan); - PartitionMetrics.bindDatabaseReplicationFactorMetricsWhenUpdate( - MetricService.getInstance(), - databaseSchemaPlan.getSchema().getName(), - databaseSchemaPlan.getSchema().getDataReplicationFactor(), - databaseSchemaPlan.getSchema().getSchemaReplicationFactor()); + if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (databaseSchema.isSetNeedLastCache() + && needInvalidateLastCache(originalSchema, databaseSchema)) { + invalidateLastCache(databaseSchema.getName()); + } + PartitionMetrics.bindDatabaseReplicationFactorMetricsWhenUpdate( + MetricService.getInstance(), + databaseSchemaPlan.getSchema().getName(), + databaseSchemaPlan.getSchema().getDataReplicationFactor(), + databaseSchemaPlan.getSchema().getSchemaReplicationFactor()); + } return result; } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); @@ -380,6 +404,7 @@ public TShowDatabaseResp showDatabase(final GetDatabasePlan getDatabasePlan) { databaseInfo.setDataReplicationFactor(databaseSchema.getDataReplicationFactor()); databaseInfo.setTimePartitionOrigin(databaseSchema.getTimePartitionOrigin()); databaseInfo.setTimePartitionInterval(databaseSchema.getTimePartitionInterval()); + databaseInfo.setNeedLastCache(isNeedLastCacheEnabled(databaseSchema)); databaseInfo.setMaxSchemaRegionNum( getMaxRegionGroupNum(database, TConsensusGroupType.SchemaRegion)); databaseInfo.setMaxDataRegionNum( @@ -902,6 +927,10 @@ public static TSStatus enrichDatabaseSchemaWithDefaultProperties( "Failed to create database. The timePartitionInterval should be positive."); } + if (!databaseSchema.isSetNeedLastCache()) { + databaseSchema.setNeedLastCache(true); + } + if (isSystemDatabase || isAuditDatabase) { databaseSchema.setMinSchemaRegionGroupNum(1); } else if (!databaseSchema.isSetMinSchemaRegionGroupNum()) { @@ -1716,20 +1745,62 @@ public synchronized Pair updateTableProperties( return new Pair<>(RpcUtils.SUCCESS_STATUS, null); } + final TDatabaseSchema databaseSchema; + try { + databaseSchema = + updatedProperties.containsKey(TsTable.NEED_LAST_CACHE_PROPERTY) + && Objects.isNull(updatedProperties.get(TsTable.NEED_LAST_CACHE_PROPERTY)) + ? getDatabaseSchemaByName(database) + : null; + } catch (final DatabaseNotExistsException e) { + throw new MetadataException(e); + } + final TsTable updatedTable = new TsTable(originalTable); updatedProperties.forEach( (k, v) -> { originalProperties.put(k, originalTable.getPropValue(k).orElse(null)); if (Objects.nonNull(v)) { updatedTable.addProp(k, v); + } else if (TsTable.NEED_LAST_CACHE_PROPERTY.equals(k) + && Objects.nonNull(databaseSchema) + && databaseSchema.isSetNeedLastCache()) { + updatedTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache())); } else { updatedTable.removeProp(k); } }); - return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable); } + private static TDatabaseSchema deepCopyDatabaseSchema(final TDatabaseSchema schema) { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ThriftConfigNodeSerDeUtils.serializeTDatabaseSchema(schema, outputStream); + return ThriftConfigNodeSerDeUtils.deserializeTDatabaseSchema( + ByteBuffer.wrap(outputStream.toByteArray())); + } + + private void invalidateLastCache(final String database) { + final Map dataNodeLocationMap = + getNodeManager().getRegisteredDataNodeLocations(); + final DataNodeAsyncRequestContext clientHandler = + new DataNodeAsyncRequestContext<>( + CnToDnAsyncRequestType.INVALIDATE_LAST_CACHE, database, dataNodeLocationMap); + CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); + clientHandler + .getResponseMap() + .forEach( + (dataNodeId, status) -> { + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.warn( + "Failed to invalidate last cache of database {} on DataNode {}, status: {}", + database, + dataNodeId, + status); + } + }); + } + public static Optional> checkTable4View( final String database, final TsTable table, final boolean isView) { if (!isView && TreeViewSchema.isTreeViewTable(table)) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index b4e2f349f015a..c086ba72ba6be 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -141,6 +141,7 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE_PATTERN; import static org.apache.iotdb.commons.schema.table.Audit.TABLE_MODEL_AUDIT_DATABASE; import static org.apache.iotdb.commons.schema.table.Audit.TREE_MODEL_AUDIT_DATABASE; +import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROPERTY; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; /** @@ -260,6 +261,21 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) { currentSchema.getTTL()); } + if (alterSchema.isSetNeedLastCache() + && alterSchema.isNeedLastCache() + != (!currentSchema.isSetNeedLastCache() || currentSchema.isNeedLastCache())) { + if (!currentSchema.isIsTableModel()) { + result.setCode(TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + result.setMessage("The tree model database does not support alter need last cache now."); + return result; + } + currentSchema.setNeedLastCache(alterSchema.isNeedLastCache()); + LOGGER.info( + "[SetNeedLastCache] The need last cache flag of Database: {} is adjusted to: {}", + currentSchema.getName(), + currentSchema.isNeedLastCache()); + } + mTree .getDatabaseNodeByDatabasePath(partialPathName) .getAsMNode() @@ -1276,6 +1292,11 @@ public ShowTableResp showTables(final ShowTablePlan plan) { TreeViewSchema.isTreeViewTable(pair.getLeft()) ? TableType.VIEW_FROM_TREE.ordinal() : TableType.BASE_TABLE.ordinal()); + info.setNeedLastCache( + pair.getLeft() + .getPropValue(NEED_LAST_CACHE_PROPERTY) + .map(Boolean::parseBoolean) + .orElse(true)); return info; }) .collect(Collectors.toList()) @@ -1324,6 +1345,11 @@ public ShowTable4InformationSchemaResp showTables4InformationSchema() { TreeViewSchema.isTreeViewTable(pair.getLeft()) ? TableType.VIEW_FROM_TREE.ordinal() : TableType.BASE_TABLE.ordinal()); + info.setNeedLastCache( + pair.getLeft() + .getPropValue(NEED_LAST_CACHE_PROPERTY) + .map(Boolean::parseBoolean) + .orElse(true)); return info; }) .collect(Collectors.toList())))); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index cf8d9d5e2ed3a..8640543d15478 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -947,6 +947,11 @@ public void setTableProperties( && databaseNode.getDatabaseSchema().isSetTTL() && databaseNode.getDatabaseSchema().getTTL() != Long.MAX_VALUE) { table.addProp(k, String.valueOf(databaseNode.getDatabaseSchema().getTTL())); + } else if (k.equals(TsTable.NEED_LAST_CACHE_PROPERTY) + && databaseNode.getDatabaseSchema().isSetNeedLastCache()) { + table.addProp( + TsTable.NEED_LAST_CACHE_PROPERTY, + String.valueOf(databaseNode.getDatabaseSchema().isNeedLastCache())); } else { table.removeProp(k); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index 05e0facb3018e..29cb315751e4c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -133,6 +133,10 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { && schema.getTTL() != Long.MAX_VALUE) { table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL())); } + if (!table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent() + && schema.isSetNeedLastCache()) { + table.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, String.valueOf(schema.isNeedLastCache())); + } setNextState(CreateTableState.PRE_CREATE); } } catch (final MetadataException | DatabaseNotExistsException e) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index 88a163157293a..ca45a9c3bc6ed 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -101,6 +101,10 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { && schema.getTTL() != Long.MAX_VALUE) { table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL())); } + if (!table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent() + && schema.isSetNeedLastCache()) { + table.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, String.valueOf(schema.isNeedLastCache())); + } setNextState(CreateTableState.PRE_CREATE); } catch (final MetadataException | DatabaseNotExistsException e) { setFailure(new ProcedureException(e)); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java index 261f60ed65a1d..2a9459daaf353 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.manager; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.junit.Assert; import org.junit.Test; @@ -37,4 +38,18 @@ public void testCalcMaxRegionGroupNum() { // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor) Assert.assertEquals(20, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0, 120, 2, 3, 5)); } + + @Test + public void testNeedLastCacheDefaultsToTrueWhenUnset() { + final TDatabaseSchema unsetSchema = new TDatabaseSchema(); + Assert.assertTrue(ClusterSchemaManager.isNeedLastCacheEnabled(unsetSchema)); + + final TDatabaseSchema enabledSchema = new TDatabaseSchema(); + enabledSchema.setNeedLastCache(true); + Assert.assertTrue(ClusterSchemaManager.isNeedLastCacheEnabled(enabledSchema)); + + final TDatabaseSchema disabledSchema = new TDatabaseSchema(); + disabledSchema.setNeedLastCache(false); + Assert.assertFalse(ClusterSchemaManager.isNeedLastCacheEnabled(disabledSchema)); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index a501824850d13..d5fa6f7d32dca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -350,6 +350,7 @@ protected void constructLine() { columnBuilders[6].writeInt(currentDatabase.getMaxSchemaRegionNum()); columnBuilders[7].writeInt(currentDatabase.getDataRegionNum()); columnBuilders[8].writeInt(currentDatabase.getMaxDataRegionNum()); + columnBuilders[9].writeBoolean(currentDatabase.isNeedLastCache()); resultBuilder.declarePosition(); currentDatabase = null; } @@ -404,6 +405,7 @@ private TableSupplier(final List dataTypes, final UserEntity userEnt table.getTableName(), table.getPropValue(TTL_PROPERTY).orElse(TTL_INFINITE)); info.setState(TableNodeStatus.USING.ordinal()); + info.setNeedLastCache(false); return info; }) .collect(Collectors.toList())); @@ -438,6 +440,7 @@ protected void constructLine() { columnBuilders[5].writeBinary( new Binary(TableType.BASE_TABLE.getName(), TSFileConfig.STRING_CHARSET)); } + columnBuilders[6].writeBoolean(currentTable.isNeedLastCache()); resultBuilder.declarePosition(); currentTable = null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index cba3f22773fe3..d1efb1c8a2634 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -331,6 +331,10 @@ public SchemaPartition getSchemaPartition( return getOrCreateSchemaPartition(database, deviceIDs, false, null); } + public boolean needLastCache(final String database) { + return partitionCache.isNeedLastCache(database); + } + private SchemaPartition getOrCreateSchemaPartition( final String database, final @Nullable List deviceIDs, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java index 0549ec3964743..99f1b9f99c3b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.file.metadata.IDeviceID; import javax.annotation.Nullable; @@ -123,6 +124,7 @@ SchemaPartition getOrCreateSchemaPartition( * *

The device id shall be [table, seg1, ....] */ + @TableModel SchemaPartition getSchemaPartition( final String database, final @Nullable List deviceIDs); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 97121eb82e86e..a7bdd5d0a1860 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaUtils; import org.apache.iotdb.db.service.metrics.CacheMetrics; import org.apache.iotdb.rpc.TSStatusCode; @@ -67,6 +68,8 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.thrift.TException; +import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.annotations.TreeModel; import org.apache.tsfile.file.metadata.IDeviceID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,6 +95,10 @@ public class PartitionCache { IoTDBDescriptor.getInstance().getMemoryConfig(); private static final List ROOT_PATH = Arrays.asList("root", "**"); + private static boolean isNeedLastCacheEnabled(final TDatabaseSchema databaseSchema) { + return !databaseSchema.isSetNeedLastCache() || databaseSchema.isNeedLastCache(); + } + /** calculate slotId by device */ private final String seriesSlotExecutorName = config.getSeriesPartitionExecutorClass(); @@ -99,7 +106,7 @@ public class PartitionCache { private final SeriesPartitionExecutor partitionExecutor; /** the cache of database */ - private final Set databaseCache = new HashSet<>(); + private final Map database2NeedLastCacheCache = new HashMap<>(); /** database -> schemaPartitionTable */ private final Cache schemaPartitionCache; @@ -200,7 +207,7 @@ public void put(final IDeviceID device, final String databaseName) { * @return database name, return {@code null} if cache miss */ private String getDatabaseName(final IDeviceID deviceID) { - for (final String database : databaseCache) { + for (final String database : database2NeedLastCacheCache.keySet()) { if (PathUtils.isStartWith(deviceID, database)) { return database; } @@ -217,7 +224,7 @@ private String getDatabaseName(final IDeviceID deviceID) { private boolean containsDatabase(final String database) { databaseCacheLock.readLock().lock(); try { - return databaseCache.contains(database); + return database2NeedLastCacheCache.containsKey(database); } finally { databaseCacheLock.readLock().unlock(); } @@ -246,7 +253,7 @@ private void fetchDatabaseAndUpdateCache( if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // update all database into cache - updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap().keySet()); + updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); getDatabaseMap(result, deviceIDs, true); } } @@ -255,19 +262,39 @@ private void fetchDatabaseAndUpdateCache( } } + @TreeModel + public boolean isNeedLastCache(final String database) { + Boolean needLastCache = database2NeedLastCacheCache.get(database); + if (Objects.nonNull(needLastCache)) { + return needLastCache; + } + try { + fetchDatabaseAndUpdateCache(false); + } catch (final TException | ClientManagerException e) { + logger.warn( + "Failed to get need_last_cache info for database {}, will put cache anyway, exception: {}", + database, + e.getMessage()); + return true; + } + needLastCache = database2NeedLastCacheCache.get(database); + return Objects.isNull(needLastCache) || needLastCache; + } + /** get all database from configNode and update database cache. */ - private void fetchDatabaseAndUpdateCache() throws ClientManagerException, TException { + private void fetchDatabaseAndUpdateCache(final boolean isTableModel) + throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TGetDatabaseReq req = new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY) - .setIsTableModel(true) + .setIsTableModel(isTableModel) .setCanSeeAuditDB(true); final TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // update all database into cache - updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap().keySet()); + updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); } } finally { databaseCacheLock.writeLock().unlock(); @@ -514,13 +541,14 @@ private void getDatabaseCacheResult( } } + @TableModel public void checkAndAutoCreateDatabase( final String database, final boolean isAutoCreate, final String userName) { boolean isExisted = containsDatabase(database); if (!isExisted) { try { // try to fetch database from config node when miss - fetchDatabaseAndUpdateCache(); + fetchDatabaseAndUpdateCache(true); isExisted = containsDatabase(database); if (!isExisted && isAutoCreate) { // try to auto create database of failed device @@ -536,12 +564,34 @@ public void checkAndAutoCreateDatabase( /** * update database cache * - * @param databaseNames the database names that need to update + * @param databases the database names + */ + public void updateDatabaseCache(final Set databases) { + databaseCacheLock.writeLock().lock(); + try { + databases.forEach(database -> database2NeedLastCacheCache.put(database, true)); + } finally { + databaseCacheLock.writeLock().unlock(); + } + } + + /** + * update database cache + * + * @param databaseMap the database names and need last cache that need to update */ - public void updateDatabaseCache(final Set databaseNames) { + public void updateDatabaseCache(final Map databaseMap) { databaseCacheLock.writeLock().lock(); try { - databaseCache.addAll(databaseNames); + databaseMap.forEach( + (database, schema) -> { + final boolean needLastCache = isNeedLastCacheEnabled(schema); + final Boolean previousNeedLastCache = + database2NeedLastCacheCache.put(database, needLastCache); + if (Boolean.TRUE.equals(previousNeedLastCache) && !needLastCache) { + TreeDeviceSchemaCacheManager.getInstance().invalidateDatabaseLastCache(database); + } + }); } finally { databaseCacheLock.writeLock().unlock(); } @@ -551,7 +601,7 @@ public void updateDatabaseCache(final Set databaseNames) { public void removeFromDatabaseCache() { databaseCacheLock.writeLock().lock(); try { - databaseCache.clear(); + database2NeedLastCacheCache.clear(); } finally { databaseCacheLock.writeLock().unlock(); } @@ -1076,7 +1126,7 @@ public void invalidAllCache() { public String toString() { return "PartitionCache{" + ", databaseCache=" - + databaseCache + + database2NeedLastCacheCache + ", replicaSetCache=" + groupIdToReplicaSetMap + ", schemaPartitionCache=" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 54ddb4ebd9da8..33ba6e4b584d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -37,6 +37,7 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Literal; @@ -288,9 +289,11 @@ import static org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted; import static org.apache.iotdb.commons.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; import static org.apache.iotdb.commons.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; +import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROPERTY; import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AbstractDatabaseTask.NEED_LAST_CACHE_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_DATA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_SCHEMA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TIME_PARTITION_INTERVAL_KEY; @@ -365,6 +368,11 @@ private IConfigTask visitDatabaseStatement( schema.setTTL(Long.MAX_VALUE); } break; + case NEED_LAST_CACHE_KEY: + if (node.getType() == DatabaseSchemaStatement.DatabaseSchemaStatementType.ALTER) { + schema.setNeedLastCache(true); + } + break; default: throw new SemanticException( DataNodeQueryMessages.UNSUPPORTED_DATABASE_PROPERTY_KEY + key); @@ -400,6 +408,9 @@ private IConfigTask visitDatabaseStatement( schema.setMaxDataRegionGroupNum( parseIntFromLiteral(value, MAX_DATA_REGION_GROUP_NUM_KEY)); break; + case NEED_LAST_CACHE_KEY: + schema.setNeedLastCache(parseBooleanFromLiteral(value, NEED_LAST_CACHE_KEY)); + break; default: throw new SemanticException( DataNodeQueryMessages.UNSUPPORTED_DATABASE_PROPERTY_KEY + key); @@ -918,17 +929,26 @@ private Map convertPropertiesToMap( if (TABLE_ALLOWED_PROPERTIES.contains(key)) { if (!property.isSetToDefault()) { final Expression value = property.getNonDefaultValue(); - final Optional strValue = parseStringFromLiteralIfBinary(value); - if (strValue.isPresent()) { - if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) { - throw new SemanticException( - "ttl value must be 'INF' or a long literal, but now is: " + value); - } - map.put(key, strValue.get().toUpperCase(Locale.ENGLISH)); - continue; + switch (key) { + case TTL_PROPERTY: + final Optional strValue = parseStringFromLiteralIfBinary(value); + if (strValue.isPresent()) { + if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) { + throw new SemanticException( + "ttl value must be 'INF' or a long literal, but now is: " + value); + } + map.put(key, strValue.get().toUpperCase(Locale.ENGLISH)); + continue; + } + map.put(key, String.valueOf(parseLongFromLiteral(value, TTL_PROPERTY))); + break; + case NEED_LAST_CACHE_PROPERTY: + map.put( + key, String.valueOf(parseBooleanFromLiteral(value, NEED_LAST_CACHE_PROPERTY))); + break; + default: + break; } - // TODO: support validation for other properties - map.put(key, String.valueOf(parseLongFromLiteral(value, TTL_PROPERTY))); } else if (serializeDefault) { map.put(key, null); } @@ -1105,6 +1125,18 @@ public IConfigTask visitSetSystemStatus(SetSystemStatus node, MPPQueryContext co return new SetSystemStatusTask(((SetSystemStatusStatement) node.getInnerTreeStatement())); } + private boolean parseBooleanFromLiteral(final Object value, final String name) { + if (!(value instanceof BooleanLiteral)) { + throw new SemanticException( + name + + " value must be a BooleanLiteral, but now is " + + (Objects.nonNull(value) ? value.getClass().getSimpleName() : null) + + ", value: " + + value); + } + return ((BooleanLiteral) value).getValue(); + } + private Optional parseStringFromLiteralIfBinary(final Object value) { return value instanceof Literal && ((Literal) value).getTsValue() instanceof Binary ? Optional.of( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java index 31c96887600c5..581b1ed2b9980 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java @@ -73,6 +73,9 @@ public static TDatabaseSchema constructDatabaseSchema( if (databaseSchemaStatement.getMaxDataRegionGroupNum() != null) { databaseSchema.setMaxDataRegionGroupNum(databaseSchemaStatement.getMaxDataRegionGroupNum()); } + if (databaseSchemaStatement.isSetNeedLastCache()) { + databaseSchema.setNeedLastCache(databaseSchemaStatement.isNeedLastCache()); + } databaseSchema.setIsTableModel(false); return databaseSchema; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java index 59b50536c15ba..c1b6ba6bfc329 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java @@ -29,6 +29,7 @@ public abstract class AbstractDatabaseTask implements IConfigTask { public static final String TIME_PARTITION_INTERVAL_KEY = "time_partition_interval"; public static final String MAX_SCHEMA_REGION_GROUP_NUM_KEY = "max_schema_region_group_num"; public static final String MAX_DATA_REGION_GROUP_NUM_KEY = "max_data_region_group_num"; + public static final String NEED_LAST_CACHE_KEY = "need_last_cache"; // Deprecated public static final String SCHEMA_REPLICATION_FACTOR_KEY = "schema_replication_factor"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java index 229cf9aa5f45b..98a0d7b6feb02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java @@ -138,12 +138,16 @@ private static String getShowCreateTableSQL(final TsTable table) { if (table.getPropValue(TsTable.COMMENT_KEY).isPresent()) { builder.append(" COMMENT ").append(getString(table.getPropValue(TsTable.COMMENT_KEY).get())); } - String ttlString = table.getPropValue(TsTable.TTL_PROPERTY).orElse(TTL_INFINITE); if (ttlString.equals(TTL_INFINITE)) { ttlString = "'" + ttlString + "'"; } - builder.append(" WITH (ttl=").append(ttlString).append(")"); + builder + .append(" WITH (ttl=") + .append(ttlString) + .append(", need_last_cache=") + .append(table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).orElse("true")) + .append(")"); return builder.toString(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java index 1882df8410b2c..dbfe6066ee2a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java @@ -147,7 +147,12 @@ public static String getShowCreateViewSQL(final TsTable table) { if (ttlString.equals(TTL_INFINITE)) { ttlString = "'" + ttlString + "'"; } - builder.append(" WITH (ttl=").append(ttlString).append(")"); + builder + .append(" WITH (ttl=") + .append(ttlString) + .append(", need_last_cache=") + .append(table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).orElse("true")) + .append(")"); builder.append(" AS "); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java index 128458888dc97..05999e3f69dce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java @@ -169,6 +169,7 @@ private static void buildTSBlockForDetails( builder.getColumnBuilder(6).writeInt(storageGroupInfo.getMaxSchemaRegionNum()); builder.getColumnBuilder(7).writeInt(storageGroupInfo.getDataRegionNum()); builder.getColumnBuilder(8).writeInt(storageGroupInfo.getMaxDataRegionNum()); + builder.getColumnBuilder(9).writeBoolean(storageGroupInfo.isNeedLastCache()); builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java index 228b1dd266acb..2ddf07f9e0ff2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java @@ -104,6 +104,9 @@ public static void buildTsBlock( ? TableType.values()[tableInfo.getType()].getName() : TableType.BASE_TABLE.getName(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(5) + .writeBoolean(!tableInfo.isSetNeedLastCache() || tableInfo.isNeedLastCache()); builder.declarePosition(); }); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index b9e2fe21c7a54..79215272046ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -2992,6 +2992,13 @@ private void parseDatabaseAttributesClause( ctx.databaseAttributeClause()) { final IoTDBSqlParser.DatabaseAttributeKeyContext attributeKey = attribute.databaseAttributeKey(); + if (attributeKey == null) { + if (attribute.NEED_LAST_CACHE() != null) { + databaseSchemaStatement.setNeedLastCache( + Boolean.parseBoolean(attribute.boolean_literal().getText())); + } + continue; + } if (attributeKey.TTL() != null) { final long ttl = Long.parseLong(attribute.INTEGER_LITERAL().getText()); databaseSchemaStatement.setTtl(ttl); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 60b6b6ebc6181..e49083e09ff2b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -66,6 +66,7 @@ import org.apache.iotdb.udf.api.relational.ScalarFunction; import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.Type; @@ -1573,6 +1574,7 @@ public SchemaPartition getOrCreateSchemaPartition( return partitionFetcher.getOrCreateSchemaPartition(database, deviceIDList, userName); } + @TableModel @Override public SchemaPartition getSchemaPartition( final String database, final List deviceIDList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index 8e40298e115bd..153bd42644d99 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; @@ -237,8 +238,9 @@ public void initOrInvalidateLastCache( readWriteLock.readLock().lock(); try { // Avoid stale table - if (Objects.isNull( - DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName(), false))) { + final TsTable table = + DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName(), false); + if (Objects.isNull(table) || Boolean.FALSE.equals(table.getCachedNeedLastCache())) { return; } dualKeyCache.update( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java index c2588606cea69..2b1b1e5ea7765 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.exception.metadata.view.InsertNonWritableViewException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; @@ -338,6 +339,10 @@ public void updateLastCacheIfExists( final @Nonnull TimeValuePair[] timeValuePairs, final boolean isAligned, final IMeasurementSchema[] measurementSchemas) { + if (!ClusterPartitionFetcher.getInstance().needLastCache(database)) { + return; + } + tableDeviceSchemaCache.updateLastCache( database, deviceID, measurements, timeValuePairs, isAligned, measurementSchemas, false); } @@ -363,6 +368,10 @@ public void updateLastCacheIfExists( * @param measurementPath the fetched {@link MeasurementPath} */ public void declareLastCache(final String database, final MeasurementPath measurementPath) { + if (!ClusterPartitionFetcher.getInstance().needLastCache(database)) { + return; + } + tableDeviceSchemaCache.updateLastCache( database, measurementPath.getIDeviceID(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java index e27dde87b41b8..dc18d56055649 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java @@ -39,6 +39,8 @@ public class DatabaseSchemaStatement extends Statement implements IConfigStateme private Integer maxSchemaRegionGroupNum = null; private Integer maxDataRegionGroupNum = null; private boolean enablePrintExceptionLog = true; + private boolean needLastCache = true; + private boolean isNeedLastCacheSet = false; // Deprecated private Integer schemaReplicationFactor = null; @@ -118,6 +120,19 @@ public void setEnablePrintExceptionLog(final boolean enablePrintExceptionLog) { this.enablePrintExceptionLog = enablePrintExceptionLog; } + public boolean isNeedLastCache() { + return needLastCache; + } + + public boolean isSetNeedLastCache() { + return isNeedLastCacheSet; + } + + public void setNeedLastCache(final boolean needLastCache) { + this.needLastCache = needLastCache; + this.isNeedLastCacheSet = true; + } + @Override public R accept(final StatementVisitor visitor, final C context) { switch (subType) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java index 2367aa0c31264..0167480576280 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java @@ -104,6 +104,7 @@ public void buildTSBlock( builder.getColumnBuilder(6).writeInt(databaseInfo.getMaxSchemaRegionNum()); builder.getColumnBuilder(7).writeInt(databaseInfo.getDataRegionNum()); builder.getColumnBuilder(8).writeInt(databaseInfo.getMaxDataRegionNum()); + builder.getColumnBuilder(9).writeBoolean(databaseInfo.isNeedLastCache()); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 11e80f5a63e8c..953de76e8a057 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; @@ -144,6 +145,12 @@ public void preUpdateTable(String database, final TsTable table, final String ol } }); LOGGER.info(DataNodeSchemaMessages.PRE_UPDATE_TABLE_SUCCESS, database, table.getTableName()); + // Since a pre-updated table can be used for query planning before commit-release, stale + // last cache should stop serving as soon as need_last_cache turns off. + final TsTable oldTableBeforeUpdate = getTableFromCache(database, table.getTableName()); + if (Objects.nonNull(oldTableBeforeUpdate) && !table.getCachedNeedLastCache()) { + TableDeviceSchemaCache.getInstance().invalidateLastCache(database, table.getTableName()); + } // If rename table if (Objects.nonNull(oldName)) { @@ -228,6 +235,21 @@ private void removeTableFromPreUpdateMap(final String database, final String tab return Objects.nonNull(tableVersionPair) ? tableVersionPair.getLeft() : null; } + private @Nullable TsTable getTableFromCache(final String database, final String tableName) { + final Map tableMap = databaseTableMap.get(database); + return Objects.nonNull(tableMap) ? tableMap.get(tableName) : null; + } + + private void invalidateLastCacheIfDisabled( + final String database, final String tableName, final @Nullable TsTable oldTable) { + final TsTable currentTable = getTableFromCache(database, tableName); + if (Objects.nonNull(oldTable) + && Objects.nonNull(currentTable) + && !currentTable.getCachedNeedLastCache()) { + TableDeviceSchemaCache.getInstance().invalidateLastCache(database, tableName); + } + } + @Override public void commitUpdateTable( String database, final String tableName, final @Nullable String oldName) { @@ -255,6 +277,7 @@ public void commitUpdateTable( databaseTableMap .computeIfAbsent(database, k -> new ConcurrentHashMap<>()) .put(tableName, newTable); + invalidateLastCacheIfDisabled(database, tableName, oldTable); if (LOGGER.isDebugEnabled()) { LOGGER.debug( DataNodeSchemaMessages.COMMIT_UPDATE_TABLE_SUCCESS_WITH_DETAIL, @@ -460,10 +483,12 @@ private void updateTable( DataNodeSchemaMessages.UPDATE_TABLE_BY_FETCH, database, tableName); } existingPair.setLeft(null); + final TsTable oldTable = getTableFromCache(database, tableName); if (Objects.nonNull(tsTable)) { databaseTableMap .computeIfAbsent(database, k -> new ConcurrentHashMap<>()) .put(tableName, tsTable); + invalidateLastCacheIfDisabled(database, tableName, oldTable); } else if (databaseTableMap.containsKey(database)) { databaseTableMap.get(database).remove(tableName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java index 443352ae919c3..50d36d5a98133 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java @@ -75,10 +75,11 @@ public static void buildDatabaseTsBlock( builder.getColumnBuilder(4).appendNull(); if (details) { for (int columnIndex = 5; - columnIndex < builder.getValueColumnBuilders().length; + columnIndex < builder.getValueColumnBuilders().length - 1; columnIndex++) { builder.getColumnBuilder(columnIndex).appendNull(); } + builder.getColumnBuilder(builder.getValueColumnBuilders().length - 1).writeBoolean(false); } builder.declarePosition(); } @@ -125,6 +126,7 @@ public static boolean mayShowTable( builder .getColumnBuilder(4) .writeBinary(new Binary(TableType.SYSTEM_VIEW.getName(), TSFileConfig.STRING_CHARSET)); + builder.getColumnBuilder(5).writeBoolean(false); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java index e08a2b610c996..6bffb381b1eab 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -53,6 +54,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class PartitionCacheTest { @@ -62,7 +64,7 @@ public class PartitionCacheTest { SeriesPartitionExecutor.getSeriesPartitionExecutor( config.getSeriesPartitionExecutorClass(), config.getSeriesPartitionSlotNum()); - private static final Set storageGroups = new HashSet<>(); + private static final Set databases = new HashSet<>(); private static final Map> schemaPartitionTable = new HashMap<>(); private static final Map< @@ -85,7 +87,7 @@ public class PartitionCacheTest { storageGroupNumber++) { // init each database String storageGroupName = getDatabaseName(storageGroupNumber); - storageGroups.add(storageGroupName); + databases.add(storageGroupName); if (!schemaPartitionTable.containsKey(storageGroupName)) { schemaPartitionTable.put(storageGroupName, new HashMap<>()); } @@ -148,7 +150,7 @@ private static String getDeviceName(String storageGroupName, int deviceNumber) { @Before public void setUp() throws Exception { partitionCache = new PartitionCache(); - partitionCache.updateDatabaseCache(storageGroups); + partitionCache.updateDatabaseCache(databases); partitionCache.updateSchemaPartitionCache(schemaPartitionTable); partitionCache.updateDataPartitionCache(dataPartitionTable); partitionCache.updateGroupIdToReplicaSetMap(100, consensusGroupIdToRegionReplicaSet); @@ -243,6 +245,29 @@ public void testStorageGroupCache() { assertEquals(0, deviceToStorageGroupMap.size()); } + @Test + public void testNeedLastCacheDefaultsToTrueWhenUnset() { + final Map databaseSchemaMap = new HashMap<>(); + + final TDatabaseSchema unsetSchema = new TDatabaseSchema(); + databaseSchemaMap.put("root.unset", unsetSchema); + + final TDatabaseSchema enabledSchema = new TDatabaseSchema(); + enabledSchema.setNeedLastCache(true); + databaseSchemaMap.put("root.enabled", enabledSchema); + + final TDatabaseSchema disabledSchema = new TDatabaseSchema(); + disabledSchema.setNeedLastCache(false); + databaseSchemaMap.put("root.disabled", disabledSchema); + + partitionCache.updateDatabaseCache(databaseSchemaMap); + + assertTrue(partitionCache.isNeedLastCache("root.unset")); + assertTrue(partitionCache.isNeedLastCache("root.enabled")); + assertFalse(partitionCache.isNeedLastCache("root.disabled")); + assertTrue(partitionCache.isNeedLastCache("root.missing")); + } + @Test public void testRegionReplicaSetCache() { // test update regionReplicaSetCache with small timestamp diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTaskTest.java new file mode 100644 index 0000000000000..41703a4583493 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTaskTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaStatement; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DatabaseSchemaTaskTest { + + @Test + public void testConstructDatabaseSchemaDoesNotSetNeedLastCacheWhenAbsent() throws Exception { + final DatabaseSchemaStatement statement = + new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.ALTER); + statement.setDatabasePath(new PartialPath("root.sg")); + + final TDatabaseSchema databaseSchema = DatabaseSchemaTask.constructDatabaseSchema(statement); + + assertFalse(databaseSchema.isSetNeedLastCache()); + } + + @Test + public void testConstructDatabaseSchemaSetsNeedLastCacheWhenPresent() throws Exception { + final DatabaseSchemaStatement statement = + new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.ALTER); + statement.setDatabasePath(new PartialPath("root.sg")); + statement.setNeedLastCache(false); + + final TDatabaseSchema databaseSchema = DatabaseSchemaTask.constructDatabaseSchema(statement); + + assertTrue(databaseSchema.isSetNeedLastCache()); + assertEquals(false, databaseSchema.isNeedLastCache()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index e808652a42a9c..6d4b072211b77 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -26,8 +26,10 @@ import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.partition.PartitionCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.tsfile.common.conf.TSFileConfig; @@ -67,6 +69,7 @@ public class TableDeviceSchemaCacheTest { private static final String database2 = "sg2"; private static final String table1 = "t1"; private static final String table2 = "t2"; + private static final String tableNeedLastCacheDisabled = "t_need_last_cache_disabled"; private static final String attributeName1 = "type"; private static final String attributeName2 = "cycle"; private static final String measurement1 = "s0"; @@ -146,6 +149,40 @@ public static void prepareEnvironment() { DataNodeTableCache.getInstance().preUpdateTable(database1, testTable2, null); DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null); + final TsTable testTableNeedLastCacheDisabled = new TsTable(tableNeedLastCacheDisabled); + columnHeaderList.forEach( + columnHeader -> + testTableNeedLastCacheDisabled.addColumnSchema( + new TagColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType()))); + testTableNeedLastCacheDisabled.addColumnSchema( + new AttributeColumnSchema(attributeName1, TSDataType.STRING)); + testTableNeedLastCacheDisabled.addColumnSchema( + new AttributeColumnSchema(attributeName2, TSDataType.STRING)); + testTableNeedLastCacheDisabled.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement1, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement2, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement3, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement4, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement5, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addColumnSchema( + new FieldColumnSchema( + measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); + testTableNeedLastCacheDisabled.addProp( + TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); + DataNodeTableCache.getInstance() + .preUpdateTable(database1, testTableNeedLastCacheDisabled, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, tableNeedLastCacheDisabled, null); + originMemConfig = memoryConfig.getSchemaCacheMemoryManager().getTotalMemorySizeInBytes(); changeSchemaCacheMemorySize(1300L); } @@ -592,6 +629,92 @@ public void testUpdateNonExistWhenWriting() { cache.getLastEntry(database1, convertTagValuesToDeviceID(table1, device0), "s2")); } + @Test + public void testLastCacheIsInvalidatedWhenTableNeedLastCacheTurnsFalse() { + final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; + final IDeviceID deviceID = convertTagValuesToDeviceID(table1, device0); + final String[] measurements = new String[] {measurement1}; + final TimeValuePair[] values = + new TimeValuePair[] {new TimeValuePair(1L, new TsPrimitiveType.TsInt(1))}; + + final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + updateLastCache4Query(cache, database1, deviceID, measurements, values); + Assert.assertEquals(values[0], cache.getLastEntry(database1, deviceID, measurement1)); + + final TsTable disabledTable = + new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1)); + disabledTable.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); + DataNodeTableCache.getInstance().preUpdateTable(database1, disabledTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + + try { + Assert.assertNull(cache.getLastEntry(database1, deviceID, measurement1)); + } finally { + final TsTable enabledTable = new TsTable(disabledTable); + enabledTable.removeProp(TsTable.NEED_LAST_CACHE_PROPERTY); + DataNodeTableCache.getInstance().preUpdateTable(database1, enabledTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + } + } + + @Test + public void testLastCacheIsInvalidatedWhenDatabaseNeedLastCacheTurnsFalse() { + final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; + final IDeviceID deviceInDatabase1 = convertTagValuesToDeviceID(table1, device0); + final IDeviceID deviceInDatabase2 = convertTagValuesToDeviceID(table1, device0); + final String[] measurements = new String[] {measurement1}; + final TimeValuePair[] values = + new TimeValuePair[] {new TimeValuePair(1L, new TsPrimitiveType.TsInt(1))}; + + final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + updateLastCache4Query(cache, database1, deviceInDatabase1, measurements, values); + updateLastCache4Query(cache, database2, deviceInDatabase2, measurements, values); + Assert.assertEquals(values[0], cache.getLastEntry(database1, deviceInDatabase1, measurement1)); + Assert.assertEquals(values[0], cache.getLastEntry(database2, deviceInDatabase2, measurement1)); + + final PartitionCache partitionCache = new PartitionCache(); + try { + final TDatabaseSchema enabledSchema = new TDatabaseSchema(); + enabledSchema.setNeedLastCache(true); + partitionCache.updateDatabaseCache(Collections.singletonMap(database1, enabledSchema)); + Assert.assertEquals( + values[0], cache.getLastEntry(database1, deviceInDatabase1, measurement1)); + + final TDatabaseSchema disabledSchema = new TDatabaseSchema(); + disabledSchema.setNeedLastCache(false); + partitionCache.updateDatabaseCache(Collections.singletonMap(database1, disabledSchema)); + + Assert.assertNull(cache.getLastEntry(database1, deviceInDatabase1, measurement1)); + Assert.assertEquals( + values[0], cache.getLastEntry(database2, deviceInDatabase2, measurement1)); + } finally { + partitionCache.invalidAllCache(); + } + } + + @Test + public void testNeedLastCacheFalseSkipsInitLastCache() { + final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; + final String[] measurements = new String[] {measurement1}; + final TimeValuePair[] values = + new TimeValuePair[] {new TimeValuePair(1L, new TsPrimitiveType.TsInt(1))}; + + final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); + final IDeviceID disabledDevice = + convertTagValuesToDeviceID(tableNeedLastCacheDisabled, device0); + + cache.initOrInvalidateLastCache(database1, disabledDevice, measurements, false); + cache.updateLastCacheIfExists(database1, disabledDevice, measurements, values); + + Assert.assertNull(cache.getLastEntry(database1, disabledDevice, measurement1)); + + final IDeviceID enabledDevice = convertTagValuesToDeviceID(table1, device0); + cache.initOrInvalidateLastCache(database1, enabledDevice, measurements, false); + cache.updateLastCacheIfExists(database1, enabledDevice, measurements, values); + + Assert.assertEquals(values[0], cache.getLastEntry(database1, enabledDevice, measurement1)); + } + private void updateLastCache4Query( final TableDeviceSchemaCache cache, final String database, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index d9d6518ac0beb..57046fcdb31d9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -252,6 +252,7 @@ private ColumnHeaderConstant() { "max_schema_region_group_num"; public static final String DATA_REGION_GROUP_NUM_TABLE_MODEL = "data_region_group_num"; public static final String MAX_DATA_REGION_GROUP_NUM_TABLE_MODEL = "max_data_region_group_num"; + public static final String NEED_LAST_CACHE_TABLE_MODEL = "need_last_cache"; public static final String REGION_ID_TABLE_MODEL = "region_id"; public static final String DATANODE_ID_TABLE_MODEL = "datanode_id"; @@ -338,6 +339,7 @@ private ColumnHeaderConstant() { public static final String PRIVILEGES = "Privileges"; public static final String COMMENT = "Comment"; public static final String TABLE_TYPE = "TableType"; + public static final String NEED_LAST_CACHE = "NeedLastCache"; public static final String VIEW = "View"; public static final String CREATE_VIEW = "Create View"; @@ -425,7 +427,8 @@ private ColumnHeaderConstant() { new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32), - new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32)); + new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List showChildPathsColumnHeaders = ImmutableList.of( @@ -741,7 +744,8 @@ private ColumnHeaderConstant() { new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32), - new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32)); + new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List describeTableColumnHeaders = ImmutableList.of( @@ -790,7 +794,8 @@ private ColumnHeaderConstant() { new ColumnHeader(COLUMN_TTL, TSDataType.TEXT), new ColumnHeader(STATUS, TSDataType.TEXT), new ColumnHeader(COMMENT, TSDataType.TEXT), - new ColumnHeader(TABLE_TYPE, TSDataType.TEXT)); + new ColumnHeader(TABLE_TYPE, TSDataType.TEXT), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List LIST_USER_OR_ROLE_PRIVILEGES_COLUMN_HEADERS = ImmutableList.of( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 27a1dbdd9d2f7..0e2e020afb0ad 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -115,6 +115,10 @@ public class InformationSchema { databaseTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.MAX_DATA_REGION_GROUP_NUM_TABLE_MODEL, TSDataType.INT32)); + databaseTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.NEED_LAST_CACHE_TABLE_MODEL, TSDataType.BOOLEAN)); + databaseTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(DATABASES, databaseTable); final TsTable tableTable = new TsTable(TABLES); @@ -134,6 +138,10 @@ public class InformationSchema { ColumnHeaderConstant.COMMENT.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); tableTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.TABLE_TYPE_TABLE_MODEL, TSDataType.STRING)); + tableTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.NEED_LAST_CACHE_TABLE_MODEL, TSDataType.BOOLEAN)); + tableTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(TABLES, tableTable); final TsTable columnTable = new TsTable(COLUMNS); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 833c37a59407f..53b4db6b8a7cc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -41,8 +41,10 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -63,7 +65,10 @@ public class TsTable { public static final String TIME_COLUMN_NAME = "time"; public static final String COMMENT_KEY = "__comment"; public static final String TTL_PROPERTY = "ttl"; - public static final Set TABLE_ALLOWED_PROPERTIES = Collections.singleton(TTL_PROPERTY); + public static final String NEED_LAST_CACHE_PROPERTY = "need_last_cache"; + public static final Set TABLE_ALLOWED_PROPERTIES = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY))); private static final String OBJECT_STRING_ERROR = "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'."; protected String tableName; @@ -90,6 +95,7 @@ public class TsTable { // Cache, avoid string parsing private transient long ttlValue = Long.MIN_VALUE; + private transient Boolean needLastCache = null; private transient int tagNums = 0; private transient int fieldNum = 0; @@ -331,6 +337,14 @@ public long getTableTTL() { : Long.MAX_VALUE; } + public boolean getCachedNeedLastCache() { + if (needLastCache == null) { + needLastCache = + getPropValue(NEED_LAST_CACHE_PROPERTY).map(Boolean::parseBoolean).orElse(true); + } + return needLastCache; + } + public Map getProps() { readWriteLock.readLock().lock(); try { @@ -366,6 +380,7 @@ public void addProp(final String key, final String value) { props = new HashMap<>(); } props.put(key, value); + invalidateCachedPropValue(key); }); } @@ -376,9 +391,18 @@ public void removeProp(final String key) { return; } props.remove(key); + invalidateCachedPropValue(key); }); } + private void invalidateCachedPropValue(final String key) { + if (TTL_PROPERTY.equals(key)) { + ttlValue = Long.MIN_VALUE; + } else if (NEED_LAST_CACHE_PROPERTY.equals(key)) { + needLastCache = null; + } + } + public void serialize(final OutputStream stream) throws IOException { ReadWriteIOUtils.write(tableName, stream); ReadWriteIOUtils.write(columnSchemaMap.size(), stream); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 852460fb60e8b..2df8e189777c0 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -234,6 +234,7 @@ struct TDatabaseSchema { 9: optional i32 maxDataRegionGroupNum 10: optional i64 timePartitionOrigin 11: optional bool isTableModel + 12: optional bool needLastCache } // Schema @@ -740,6 +741,7 @@ struct TDatabaseInfo { 9: required i32 dataRegionNum 11: required i32 maxDataRegionNum 12: optional i64 timePartitionOrigin + 13: optional bool needLastCache } struct TGetDatabaseReq { @@ -1294,6 +1296,7 @@ struct TTableInfo { 3: optional i32 state 4: optional string comment 5: optional i32 type + 6: optional bool needLastCache } struct TCreateTableViewReq {