Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;

Expand All @@ -61,14 +62,16 @@ protected void setupConfig() {
.getConfig()
.getCommonConfig()
.setDataReplicationFactor(1)
.setSchemaReplicationFactor(1);
.setSchemaReplicationFactor(1)
.setPipeAutoSplitFullEnabled(true);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(false)
.setDatanodeMemoryProportion("3:3:1:1:1:0")
.setDataReplicationFactor(1)
.setSchemaReplicationFactor(1);
.setSchemaReplicationFactor(1)
.setPipeAutoSplitFullEnabled(true);
}

@Test
Expand Down Expand Up @@ -122,6 +125,63 @@ public void testReceiverAutoCreateSchemaDisabledWithSpecialTimeSeries() throws E
}
}

@Test
public void testAutoSplitHistoryTsFileWithDeletionWhenReceiverAutoCreateSchemaDisabled()
throws Exception {
TestUtils.executeNonQueries(
senderEnv,
Arrays.asList(
"create database root.sg",
"create timeseries root.sg.non_aligned.s1 with datatype=INT32",
"create timeseries root.sg.non_aligned.s2 with datatype=DOUBLE",
"create aligned timeseries root.sg.aligned(s1 INT32, s2 DOUBLE)",
"create timeseries root.sg.deleted_measurement.s1 with datatype=INT32",
"create timeseries root.sg.deleted_measurement.s2 with datatype=DOUBLE",
"insert into root.sg.non_aligned(time, s1, s2) values(1, 1, 1.0), (2, 2, 2.0), (3, 3, 3.0)",
"insert into root.sg.aligned(time, s1, s2) values(1, 10, 10.0), (2, 20, 20.0), (3, 30, 30.0)",
"insert into root.sg.deleted_measurement(time, s1, s2) values(1, 100, 100.0), (2, 200, 200.0)",
"flush",
"delete from root.sg.non_aligned.s1 where time > 2",
"delete from root.sg.aligned.* where time > 2",
"delete timeseries root.sg.deleted_measurement.s1",
"flush"));

awaitUntilFlush(senderEnv);

TestUtils.executeNonQuery(
senderEnv,
String.format(
"create pipe test with source ('inclusion'='all', 'source.history.enable'='true', 'source.realtime.mode'='batch') "
+ "with sink ('sink'='iotdb-thrift-sink', 'sink.node-urls'='%s')",
receiverEnv.getDataNodeWrapper(0).getIpAndPortString()));

TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"select * from root.sg.non_aligned",
"Time,root.sg.non_aligned.s1,root.sg.non_aligned.s2,",
new HashSet<>(Arrays.asList("1,1,1.0,", "2,2,2.0,", "3,null,3.0,")));
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"select * from root.sg.aligned",
"Time,root.sg.aligned.s1,root.sg.aligned.s2,",
new HashSet<>(Arrays.asList("1,10,10.0,", "2,20,20.0,")));
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"select * from root.sg.deleted_measurement",
"Time,root.sg.deleted_measurement.s2,",
new HashSet<>(Arrays.asList("1,100.0,", "2,200.0,")));
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"count timeseries root.sg.deleted_measurement.*",
"count(timeseries),",
new HashSet<>(Arrays.asList("1,")));
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"show devices root.sg.aligned",
"Device,IsAligned,Template,TTL(ms),",
new HashSet<>(Arrays.asList("root.sg.aligned,true,null,INF,")));
}

private QueryResult queryForResult(final Statement statement, final String sql)
throws SQLException {
try (final ResultSet resultSet = statement.executeQuery(sql)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,12 @@ public final class DataNodeQueryMessages {
"Empty file detected, will skip loading this file: {}";
public static final String AUTO_CREATE_OR_VERIFY_SCHEMA_ERROR =
"Auto create or verify schema error.";
public static final String LOAD_TSFILE_DEVICE_SCHEMA_MISSING_AUTO_CREATE_DISABLED =
"Device %s does not exist in IoTDB and can not be created. Please check whether auto-create-schema is enabled.";
public static final String LOAD_TSFILE_MEASUREMENT_SCHEMA_MISSING_AUTO_CREATE_DISABLED =
"Measurement %s does not exist in IoTDB and can not be created. Please check whether auto-create-schema is enabled.";
public static final String PIPE_GENERATED_LOAD_TSFILE_WAITING_FOR_SCHEMA_METADATA =
"Pipe generated LoadTsFile is waiting for schema metadata to be transferred. Detail: %s";
public static final String FAILED_TO_FIND_TAG_COLUMN_MAPPING_FOR_TABLE =
"Failed to find tag column mapping for table {}";
public static final String AUTO_CREATE_DATABASE_FAILED_BECAUSE =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,12 @@ public final class DataNodeQueryMessages {
"检测到空文件,将跳过加载此文件:{}";
public static final String AUTO_CREATE_OR_VERIFY_SCHEMA_ERROR =
"自动创建或验证 schema 出错。";
public static final String LOAD_TSFILE_DEVICE_SCHEMA_MISSING_AUTO_CREATE_DISABLED =
"设备 %s 在 IoTDB 中不存在且无法被创建。请检查是否启用了 auto-create-schema。";
public static final String LOAD_TSFILE_MEASUREMENT_SCHEMA_MISSING_AUTO_CREATE_DISABLED =
"时间序列 %s 在 IoTDB 中不存在且无法被创建。请检查是否启用了 auto-create-schema。";
public static final String PIPE_GENERATED_LOAD_TSFILE_WAITING_FOR_SCHEMA_METADATA =
"Pipe 生成的 LoadTsFile 正在等待 schema 元数据传输完成。详情:%s";
public static final String FAILED_TO_FIND_TAG_COLUMN_MAPPING_FOR_TABLE =
"未找到表 {} 的标签列映射";
public static final String AUTO_CREATE_DATABASE_FAILED_BECAUSE =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.db.exception.load;

public class LoadAnalyzeMissingSchemaException extends LoadAnalyzeException {

public LoadAnalyzeMissingSchemaException(final String message) {
super(message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ public TSStatus visitNode(final StatementNode node, final TSStatus status) {
public TSStatus visitLoadFile(
final LoadTsFileStatement loadTsFileStatement, final TSStatus status) {
if (status.getCode() == TSStatusCode.SYSTEM_READ_ONLY.getStatusCode()
|| status.getCode() == TSStatusCode.LOAD_FILE_ERROR.getStatusCode()
&& status.getMessage() != null
&& status.getMessage().contains("memory")) {
|| status.getCode() == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode()) {
return new TSStatus(
TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode())
.setMessage(status.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.load.LoadAnalyzeException;
import org.apache.iotdb.db.exception.load.LoadAnalyzeMissingSchemaException;
import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException;
import org.apache.iotdb.db.exception.load.LoadEmptyFileException;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
Expand Down Expand Up @@ -222,6 +223,9 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) {
executeTabletConversionOnException(analysis, e);
return analysis;
} catch (Exception e) {
if (setTemporaryUnavailableStatusIfNecessary(analysis, e)) {
return analysis;
}
final String exceptionMessage =
String.format(
"Auto create or verify schema error when executing statement %s. Detail: %s.",
Expand Down Expand Up @@ -346,6 +350,9 @@ private boolean doAnalyzeFileByFile(IAnalysis analysis) {
"The file %s is not a valid tsfile. Please check the input file.",
tsFile.getPath()));
} catch (Exception e) {
if (setTemporaryUnavailableStatusIfNecessary(analysis, e)) {
return false;
}
final String exceptionMessage =
String.format(
"Loading file %s failed. Detail: %s",
Expand Down Expand Up @@ -681,8 +688,26 @@ private void setFailAnalysisForAuthException(IAnalysis analysis, AuthException e
analysis.setFailStatus(RpcUtils.getStatus(e.getCode(), e.getMessage()));
}

private void setFailAnalysisForTemporaryUnavailablePipeSchema(
final IAnalysis analysis, final Throwable throwable) {
final String exceptionMessage =
String.format(
DataNodeQueryMessages.PIPE_GENERATED_LOAD_TSFILE_WAITING_FOR_SCHEMA_METADATA,
throwable.getMessage() == null
? throwable.getClass().getName()
: throwable.getMessage());
analysis.setFinishQueryAfterAnalyze(true);
analysis.setFailStatus(
RpcUtils.getStatus(TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION, exceptionMessage));
setRealStatement(analysis);
}

private void executeTabletConversionOnException(
final IAnalysis analysis, final LoadAnalyzeException e) {
if (setTemporaryUnavailableStatusIfNecessary(analysis, e)) {
return;
}

if (shouldSkipConversion(e)) {
analysis.setFailStatus(
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
Expand Down Expand Up @@ -764,6 +789,38 @@ private void executeTabletConversionOnException(
setRealStatement(analysis);
}

private boolean setTemporaryUnavailableStatusIfNecessary(
final IAnalysis analysis, final Throwable throwable) {
if (isTemporaryUnavailableDueToPipeSchemaNotReady(throwable)) {
setFailAnalysisForTemporaryUnavailablePipeSchema(analysis, throwable);
return true;
}
if (isGeneratedByPipe && LoadTsFileDataTypeConverter.isMemoryPressureException(throwable)) {
analysis.setFinishQueryAfterAnalyze(true);
analysis.setFailStatus(LoadTsFileDataTypeConverter.getMemoryPressureStatus(throwable));
setRealStatement(analysis);
return true;
}
return false;
}

boolean isTemporaryUnavailableDueToPipeSchemaNotReady(final Throwable throwable) {
if (!isGeneratedByPipe
|| !isVerifySchema
|| IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()) {
return false;
}

Throwable current = throwable;
while (current != null) {
if (current instanceof LoadAnalyzeMissingSchemaException) {
return true;
}
current = current.getCause();
}
return false;
}

private boolean shouldSkipConversion(LoadAnalyzeException e) {
return (e instanceof LoadAnalyzeTypeMismatchException) && !isConvertOnTypeMismatch;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.load.LoadAnalyzeException;
import org.apache.iotdb.db.exception.load.LoadAnalyzeMissingSchemaException;
import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException;
import org.apache.iotdb.db.exception.load.LoadFileException;
import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException;
Expand Down Expand Up @@ -109,7 +110,7 @@ public void setCurrentTimeIndex(final ITimeIndex timeIndex) {
public void autoCreateAndVerify(
TsFileSequenceReader reader,
Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
throws IOException, AuthException, LoadAnalyzeTypeMismatchException {
throws IOException, AuthException, LoadAnalyzeException {
for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
device2TimeseriesMetadataList.entrySet()) {
final IDeviceID device = entry.getKey();
Expand Down Expand Up @@ -198,14 +199,14 @@ public void flushAndClearDeviceIsAlignedCacheIfNecessary() throws SemanticExcept
schemaCache.clearDeviceIsAlignedCacheIfNecessary();
}

public void flush() throws AuthException, LoadAnalyzeTypeMismatchException {
public void flush() throws AuthException, LoadAnalyzeException {
doAutoCreateAndVerify();

schemaCache.clearTimeSeries();
}

private void doAutoCreateAndVerify()
throws SemanticException, AuthException, LoadAnalyzeTypeMismatchException {
throws SemanticException, AuthException, LoadAnalyzeException {
if (schemaCache.getDevice2TimeSeries().isEmpty()) {
return;
}
Expand Down Expand Up @@ -235,6 +236,11 @@ private void doAutoCreateAndVerify()
} else {
handleException(e, loadTsFileAnalyzer.getStatementString());
}
} catch (LoadAnalyzeMissingSchemaException e) {
if (loadTsFileAnalyzer.isTemporaryUnavailableDueToPipeSchemaNotReady(e)) {
throw e;
}
handleException(e, loadTsFileAnalyzer.getStatementString());
} catch (Exception e) {
if (e.getCause() instanceof LoadAnalyzeTypeMismatchException
&& loadTsFileAnalyzer.isConvertOnTypeMismatch()) {
Expand Down Expand Up @@ -449,10 +455,9 @@ private void verifySchema(ISchemaTree schemaTree)
.collect(Collectors.toList()));

if (iotdbDeviceSchemaInfo == null) {
throw new LoadAnalyzeException(
throw new LoadAnalyzeMissingSchemaException(
String.format(
"Device %s does not exist in IoTDB and can not be created. "
+ "Please check weather auto-create-schema is enabled.",
DataNodeQueryMessages.LOAD_TSFILE_DEVICE_SCHEMA_MISSING_AUTO_CREATE_DISABLED,
device));
}

Expand All @@ -475,10 +480,9 @@ private void verifySchema(ISchemaTree schemaTree)
final IMeasurementSchema tsFileSchema = tsfileTimeseriesSchemas.get(i);
final IMeasurementSchema iotdbSchema = iotdbTimeseriesSchemas.get(i);
if (iotdbSchema == null) {
throw new LoadAnalyzeException(
throw new LoadAnalyzeMissingSchemaException(
String.format(
"Measurement %s does not exist in IoTDB and can not be created. "
+ "Please check weather auto-create-schema is enabled.",
DataNodeQueryMessages.LOAD_TSFILE_MEASUREMENT_SCHEMA_MISSING_AUTO_CREATE_DISABLED,
device + TsFileConstant.PATH_SEPARATOR + tsfileTimeseriesSchemas.get(i)));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@
import org.apache.iotdb.commons.utils.StatusUtils;
import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.BatchActivateTemplateStatement;
import org.apache.iotdb.rpc.TSStatusCode;

import org.junit.Assert;
import org.junit.Test;

import java.io.File;
import java.util.Arrays;
import java.util.Collections;

Expand Down Expand Up @@ -64,6 +66,36 @@ StatusUtils.OK, new TSStatus(TSStatusCode.OUT_OF_TTL.getStatusCode()))))
.getCode());
}

@Test
public void testLoadTemporaryUnavailableClassification() throws Exception {
final File tsFile = File.createTempFile("temporary-unavailable", ".tsfile");
tsFile.deleteOnExit();

Assert.assertEquals(
TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode(),
IoTDBDataNodeReceiver.STATEMENT_STATUS_VISITOR
.process(
LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()),
new TSStatus(TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode())
.setMessage("schema is not ready"))
.getCode());
}

@Test
public void testLoadFileErrorWithMemoryMessageIsNotClassifiedByMessage() throws Exception {
final File tsFile = File.createTempFile("memory-error", ".tsfile");
tsFile.deleteOnExit();

Assert.assertEquals(
TSStatusCode.LOAD_FILE_ERROR.getStatusCode(),
IoTDBDataNodeReceiver.STATEMENT_STATUS_VISITOR
.process(
LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()),
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode())
.setMessage("memory pressure"))
.getCode());
}

@Test
public void testDatabaseNotExistRuntimeExceptionClassification() {
Assert.assertEquals(
Expand Down
Loading
Loading