Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.TimeUnit;

import static org.awaitility.Awaitility.await;
Expand Down Expand Up @@ -324,7 +325,7 @@ private void prepareTypeConversionTest(
generateMeasurementSchemas();

// Generate createTimeSeries in sender and receiver
String uuid = "bcdedit";
String uuid = "bcdedit" + UUID.randomUUID().toString().replace("-", "");
for (Pair<MeasurementSchema, MeasurementSchema> pair : measurementSchemas) {
createTimeSeries(uuid, pair.left.getMeasurementName(), pair.left.getType().name(), senderEnv);
createTimeSeries(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
Expand Down Expand Up @@ -433,13 +434,14 @@ public static class ValueHider {
}

static String hide(final String key, final String value) {
if (Objects.isNull(key)) {
return value;
}
if (KEYS.contains(KeyReducer.reduce(key))) {
if (isHiddenKey(key)) {
return PLACEHOLDER;
}
return value;
}

public static boolean isHiddenKey(final String key) {
return Objects.nonNull(key) && KEYS.contains(KeyReducer.reduce(key).toLowerCase(Locale.ROOT));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ protected boolean executeOnce() throws Exception {
throw new PipeException(
String.format(
"Exception in pipe process, subtask: %s, last event: %s, root cause: %s",
taskID,
getDisplayTaskID(),
lastEvent instanceof EnrichedEvent
? ((EnrichedEvent) lastEvent).coreReportMessage()
: lastEvent,
Expand Down Expand Up @@ -300,7 +300,7 @@ public void close() {
} catch (final Exception e) {
LOGGER.info(
DataNodePipeMessages.EXCEPTION_OCCURRED_WHEN_CLOSING_PIPE_PROCESSOR_SUBTASK,
taskID,
getDisplayTaskID(),
ErrorHandlingCommonUtils.getRootCause(e).getMessage(),
e);
} finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public class PipeSinkSubtask extends PipeAbstractSinkSubtask {

// Record these variables to provide corresponding value to tag key of monitoring metrics
private final String attributeSortedString;
private final String attributeDisplayString;
private final int sinkIndex;

// Now parallel connectors run the same time, thus the heartbeat events are not sure
Expand All @@ -75,8 +76,27 @@ public PipeSinkSubtask(
final int sinkIndex,
final UnboundedBlockingPendingQueue<Event> inputPendingQueue,
final PipeConnector outputPipeConnector) {
this(
taskID,
creationTime,
attributeSortedString,
attributeSortedString,
sinkIndex,
inputPendingQueue,
outputPipeConnector);
}

public PipeSinkSubtask(
final String taskID,
final long creationTime,
final String attributeSortedString,
final String attributeDisplayString,
final int sinkIndex,
final UnboundedBlockingPendingQueue<Event> inputPendingQueue,
final PipeConnector outputPipeConnector) {
super(taskID, creationTime, outputPipeConnector);
this.attributeSortedString = attributeSortedString;
this.attributeDisplayString = attributeDisplayString;
this.sinkIndex = sinkIndex;
this.inputPendingQueue = inputPendingQueue;

Expand Down Expand Up @@ -156,7 +176,7 @@ private void transferHeartbeatEvent(final PipeHeartbeatEvent event) {
DataNodePipeMessages.PIPECONNECTOR
+ outputPipeSink.getClass().getName()
+ "(id: "
+ taskID
+ getDisplayTaskID()
+ ")"
+ " heartbeat failed, or encountered failure when transferring generic event. Failure: "
+ e.getMessage(),
Expand All @@ -181,13 +201,13 @@ public void close() {
outputPipeSink.close();
LOGGER.info(
DataNodePipeMessages.PIPE_CONNECTOR_SUBTASK_WAS_CLOSED_WITHIN_MS,
taskID,
getDisplayTaskID(),
outputPipeSink,
System.currentTimeMillis() - startTime);
} catch (final Exception e) {
LOGGER.info(
DataNodePipeMessages.EXCEPTION_OCCURRED_WHEN_CLOSING_PIPE_CONNECTOR_SUBTASK,
taskID,
getDisplayTaskID(),
ErrorHandlingCommonUtils.getRootCause(e).getMessage(),
e);
} finally {
Expand Down Expand Up @@ -366,4 +386,14 @@ protected void report(final EnrichedEvent event, final PipeRuntimeException exce
lastExceptionTime = Long.MAX_VALUE;
PipeDataNodeAgent.runtime().report(event, exception);
}

@Override
public String getDisplayTaskID() {
return generateDisplayTaskID(attributeDisplayString, creationTime, sinkIndex);
}

static String generateDisplayTaskID(
final String attributeDisplayString, final long creationTime, final int sinkIndex) {
return String.format("%s_%s_%s", attributeDisplayString, creationTime, sinkIndex);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public synchronized void register() {
registeredTaskCount++;
LOGGER.info(
DataNodePipeMessages.REGISTER_SUBTASK_RUNNINGTASKCOUNT_REGISTEREDTASKCOUNT,
subtask,
subtask.getDisplayTaskID(),
runningTaskCount,
registeredTaskCount);
}
Expand Down Expand Up @@ -112,7 +112,7 @@ public synchronized boolean deregister(final CommitterKey committerKey) {
registeredTaskCount--;
LOGGER.info(
DataNodePipeMessages.DEREGISTER_SUBTASK_RUNNINGTASKCOUNT_REGISTEREDTASKCOUNT,
subtask,
subtask.getDisplayTaskID(),
runningTaskCount,
registeredTaskCount);
}
Expand All @@ -135,7 +135,7 @@ public synchronized void start() {
runningTaskCount++;
LOGGER.info(
DataNodePipeMessages.START_SUBTASK_RUNNINGTASKCOUNT_REGISTEREDTASKCOUNT,
subtask,
subtask.getDisplayTaskID(),
runningTaskCount,
registeredTaskCount);
}
Expand All @@ -152,7 +152,7 @@ public synchronized void stop() {
runningTaskCount--;
LOGGER.info(
DataNodePipeMessages.STOP_SUBTASK_RUNNINGTASKCOUNT_REGISTEREDTASKCOUNT,
subtask,
subtask.getDisplayTaskID(),
runningTaskCount,
registeredTaskCount);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ public class PipeSinkSubtaskManager {
private final Map<String, List<PipeSinkSubtaskLifeCycle>>
attributeSortedString2SubtaskLifeCycleMap = new HashMap<>();

private final Map<String, String> attributeSortedString2DisplayString = new HashMap<>();

public synchronized String register(
final Supplier<? extends PipeSinkSubtaskExecutor> executorSupplier,
final PipeParameters pipeSinkParameters,
Expand Down Expand Up @@ -92,19 +94,14 @@ public synchronized String register(
final int sinkNum;
boolean realTimeFirst = false;
String attributeSortedString = generateAttributeSortedString(pipeSinkParameters);
final String attributeDisplayString = generateAttributeDisplayString(pipeSinkParameters);
if (isDataRegionSink) {
sinkNum =
pipeSinkParameters.getIntOrDefault(
Arrays.asList(
PipeSinkConstant.CONNECTOR_IOTDB_PARALLEL_TASKS_KEY,
PipeSinkConstant.SINK_IOTDB_PARALLEL_TASKS_KEY),
PipeSinkConstant.SINGLE_THREAD_DEFAULT_SINK.contains(
pipeSinkParameters
.getStringOrDefault(
Arrays.asList(
PipeSinkConstant.CONNECTOR_KEY, PipeSinkConstant.SINK_KEY),
BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName())
.toLowerCase())
PipeSinkConstant.SINGLE_THREAD_DEFAULT_SINK.contains(connectorKey)
? 1
: PipeSinkConstant.CONNECTOR_IOTDB_PARALLEL_TASKS_DEFAULT_VALUE);
realTimeFirst =
Expand All @@ -120,7 +117,9 @@ public synchronized String register(
sinkNum = 1;
attributeSortedString = "schema_" + attributeSortedString;
}
environment.setAttributeSortedString(attributeSortedString);
final String attributeDisplayStringWithPrefix =
isDataRegionSink ? "data_" + attributeDisplayString : "schema_" + attributeDisplayString;
environment.setAttributeSortedString(attributeDisplayStringWithPrefix);

if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
final PipeSinkSubtaskExecutor executor = executorSupplier.get();
Expand All @@ -138,6 +137,11 @@ public synchronized String register(
}

for (int sinkIndex = 0; sinkIndex < sinkNum; sinkIndex++) {
final String taskID =
String.format(
"%s_%s_%s",
attributeDisplayStringWithPrefix, environment.getCreationTime(), sinkIndex);

final PipeConnector pipeSink =
isDataRegionSink
? PipeDataNodeAgent.plugin().dataRegion().reflectSink(pipeSinkParameters)
Expand Down Expand Up @@ -168,10 +172,10 @@ public synchronized String register(
// 2. Construct PipeConnectorSubtaskLifeCycle to manage PipeConnectorSubtask's life cycle
final PipeSinkSubtask pipeSinkSubtask =
new PipeSinkSubtask(
String.format(
"%s_%s_%s", attributeSortedString, environment.getCreationTime(), sinkIndex),
taskID,
environment.getCreationTime(),
attributeSortedString,
attributeDisplayStringWithPrefix,
sinkIndex,
pendingQueue,
pipeSink);
Expand All @@ -182,11 +186,13 @@ public synchronized String register(

LOGGER.info(
DataNodePipeMessages.PIPE_SINK_SUBTASKS_WITH_ATTRIBUTES_IS_BOUNDED,
attributeSortedString,
attributeDisplayStringWithPrefix,
executor.getWorkingThreadName(),
executor.getCallbackThreadName());
attributeSortedString2SubtaskLifeCycleMap.put(
attributeSortedString, pipeSinkSubtaskLifeCycleList);
attributeSortedString2DisplayString.put(
attributeSortedString, attributeDisplayStringWithPrefix);
}

for (final PipeSinkSubtaskLifeCycle lifeCycle :
Expand All @@ -203,7 +209,7 @@ public synchronized void deregister(
final int regionId,
final String attributeSortedString) {
if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
throw new PipeException(FAILED_TO_DEREGISTER_EXCEPTION_MESSAGE + attributeSortedString);
throwNoSuchSubtaskException(attributeSortedString);
}

final List<PipeSinkSubtaskLifeCycle> lifeCycles =
Expand All @@ -219,6 +225,7 @@ public synchronized void deregister(

if (lifeCycles.isEmpty()) {
attributeSortedString2SubtaskLifeCycleMap.remove(attributeSortedString);
attributeSortedString2DisplayString.remove(attributeSortedString);
executor.shutdown();
LOGGER.info(
DataNodePipeMessages.THE_EXECUTOR_AND_HAS_BEEN_SUCCESSFULLY_SHUTDOWN,
Expand All @@ -234,7 +241,7 @@ public synchronized void deregister(

public synchronized void start(final String attributeSortedString) {
if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
throw new PipeException(FAILED_TO_DEREGISTER_EXCEPTION_MESSAGE + attributeSortedString);
throwNoSuchSubtaskException(attributeSortedString);
}

for (final PipeSinkSubtaskLifeCycle lifeCycle :
Expand All @@ -245,7 +252,7 @@ public synchronized void start(final String attributeSortedString) {

public synchronized void stop(final String attributeSortedString) {
if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
throw new PipeException(FAILED_TO_DEREGISTER_EXCEPTION_MESSAGE + attributeSortedString);
throwNoSuchSubtaskException(attributeSortedString);
}

for (final PipeSinkSubtaskLifeCycle lifeCycle :
Expand All @@ -258,7 +265,8 @@ public UnboundedBlockingPendingQueue<Event> getPipeSinkPendingQueue(
final String attributeSortedString) {
if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
throw new PipeException(
DataNodePipeMessages.FAILED_TO_GET_PENDINGQUEUE_NO_SUCH_SUBTASK + attributeSortedString);
DataNodePipeMessages.FAILED_TO_GET_PENDINGQUEUE_NO_SUCH_SUBTASK
+ getDisplayStringForException(attributeSortedString));
}

// All subtasks share the same pending queue
Expand All @@ -268,13 +276,35 @@ public UnboundedBlockingPendingQueue<Event> getPipeSinkPendingQueue(
.getPendingQueue();
}

private String generateAttributeSortedString(final PipeParameters pipeConnectorParameters) {
private static String generateAttributeSortedString(
final PipeParameters pipeConnectorParameters) {
final TreeMap<String, String> sortedStringSourceMap =
new TreeMap<>(pipeConnectorParameters.getAttribute());
sortedStringSourceMap.remove(SystemConstant.RESTART_OR_NEWLY_ADDED_KEY);
return sortedStringSourceMap.toString();
}

/**
* Attribute string for logs, metrics and exception messages with sensitive attributes removed.
*/
static String generateAttributeDisplayString(final PipeParameters pipeConnectorParameters) {
final TreeMap<String, String> filteredAttributes =
new TreeMap<>(pipeConnectorParameters.getAttribute());
filteredAttributes.remove(SystemConstant.RESTART_OR_NEWLY_ADDED_KEY);
filteredAttributes.keySet().removeIf(PipeParameters.ValueHider::isHiddenKey);
return filteredAttributes.toString();
}

private void throwNoSuchSubtaskException(final String attributeSortedString) {
throw new PipeException(
FAILED_TO_DEREGISTER_EXCEPTION_MESSAGE
+ getDisplayStringForException(attributeSortedString));
}

private String getDisplayStringForException(final String attributeSortedString) {
return attributeSortedString2DisplayString.getOrDefault(attributeSortedString, "unknown");
}

///////////////////////// Singleton Instance Holder /////////////////////////

private PipeSinkSubtaskManager() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,9 @@ public void register(final PipeSinkSubtask pipeSinkSubtask) {

public void deregister(final String taskID) {
if (!connectorMap.containsKey(taskID)) {
LOGGER.warn(DataNodePipeMessages.FAILED_TO_DEREGISTER_PIPE_SCHEMA_REGION_CONNECTOR, taskID);
LOGGER.warn(
DataNodePipeMessages.FAILED_TO_DEREGISTER_PIPE_SCHEMA_REGION_CONNECTOR,
getDisplayTaskID(taskID));
return;
}
if (Objects.nonNull(metricService)) {
Expand All @@ -125,12 +127,18 @@ public void markSchemaEvent(final String taskID) {
}
final Rate rate = schemaRateMap.get(taskID);
if (rate == null) {
LOGGER.info(DataNodePipeMessages.FAILED_TO_MARK_PIPE_SCHEMA_REGION_WRITE, taskID);
LOGGER.info(
DataNodePipeMessages.FAILED_TO_MARK_PIPE_SCHEMA_REGION_WRITE, getDisplayTaskID(taskID));
return;
}
rate.mark();
}

private String getDisplayTaskID(final String taskID) {
final PipeSinkSubtask connector = connectorMap.get(taskID);
return Objects.nonNull(connector) ? connector.getDisplayTaskID() : "unknown";
}

//////////////////////////// singleton ////////////////////////////

private static class PipeSchemaRegionSinkMetricsHolder {
Expand Down
Loading